From 17d6f54fa40311289ebacbdf376d4754e8cdbcac Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Wed, 1 Jan 2025 14:05:56 +0800 Subject: [PATCH 01/56] ggml: add s390x ARCH_FLAGS for compilation Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index 6b3641c4263c7..8658513b2dd0f 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -303,6 +303,10 @@ function(ggml_add_cpu_backend_variant_impl tag_name) if (GGML_LSX) list(APPEND ARCH_FLAGS -mlsx) endif() + elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "s390x") + message(STATUS "s390x detected") + + list(APPEND ARCH_FLAGS -march=z15 -mtune=z15 -mvx -mzvector) elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "riscv64") message(STATUS "RISC-V detected") if (GGML_RVV) From 891922f7c73bf037852d1499d9f85871c384da3c Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Wed, 1 Jan 2025 14:29:33 +0800 Subject: [PATCH 02/56] ggml: add SIMD for s390x using vector intrinsics SIMD is activated for: * ggml_vec_dot_f32 * ggml_vec_dot_f16 * ggml_vec_mad_f32 * ggml_vec_mad_f16 * ggml_vec_mad_f32_unroll * ggml_vec_scale_f32 * ggml_vec_scale_f16 SIMD is NOT activated for: * ggml_vec_dot_f16_unroll (pending bugfix) Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu.c | 115 ++++++++++++++++++++++++++++++++--- 1 file changed, 107 insertions(+), 8 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index b7fefb9ddfd89..7a9ea95a5912a 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -87,6 +87,10 @@ #include #endif +#if defined(__s390x__) && defined(__VEC__) +#include +#endif + // floating point type used to accumulate sums typedef double ggml_float; @@ -238,6 +242,8 @@ typedef pthread_t ggml_thread_t; #else #if defined(__POWER9_VECTOR__) #define CACHE_LINE_SIZE 128 +#elif defined(__s390x__) && defined(__VEC__) +#define CACHE_LINE_SIZE 256 #else #define CACHE_LINE_SIZE 64 #endif @@ -1218,11 +1224,96 @@ static inline void __lsx_f16x4_store(ggml_fp16_t * x, __m128 y) { #define GGML_F16_VEC_MUL GGML_F32Cx4_MUL #define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE +#elif defined(__s390x__) && defined(__VEC__) +#define vec_add(a, b) ((a) + (b)) +#define vec_mul(a, b) ((a) * (b)) + +// TODO: Activate this macro +//#define GGML_SIMD + +// F32 s390x + +#define GGML_F32_STEP 32 +#define GGML_F32_EPR 4 + +#define GGML_F32x4 __vector float +#define GGML_F32x4_ZERO vec_splats(0.0f) +#define GGML_F32x4_SET1 vec_splats +#define GGML_F32x4_LOAD(p) vec_xl(0, p) +#define GGML_F32x4_STORE(p, r) vec_xst(r, 0, p) +#define GGML_F32x4_FMA(a, b, c) vec_madd(b, c, a) +#define GGML_F32x4_ADD vec_add +#define GGML_F32x4_MUL vec_mul +#define GGML_F32x4_REDUCE(res, x) \ +{ \ + int offset = GGML_F32_ARR >> 1; \ + for (int i = 0; i < offset; ++i) { \ + sum[i] = vec_add(sum[i], sum[offset + i]); \ + } \ + offset >>= 1; + for (int i = 0; i < offset; ++i) { \ + sum[i] = vec_add(sum[i], sum[offset + i]); \ + } \ + offset >>= 1; + for (int i = 0; i < offset; ++i) { \ + sum[i] = vec_add(sum[i], sum[offset + i]); \ + } \ + res = vec_extract(x[0], 0) + \ + vec_extract(x[0], 1) + \ + vec_extract(x[0], 2) + \ + vec_extract(x[0], 3); \ +} + +#define GGML_F32_VEC GGML_F32x4 +#define GGML_F32_VEC_ZERO GGML_F32x4_ZERO +#define GGML_F32_VEC_SET1 GGML_F32x4_SET1 +#define GGML_F32_VEC_LOAD GGML_F32x4_LOAD +#define GGML_F32_VEC_STORE GGML_F32x4_STORE +#define GGML_F32_VEC_FMA GGML_F32x4_FMA +#define GGML_F32_VEC_ADD GGML_F32x4_ADD +#define GGML_F32_VEC_MUL GGML_F32x4_MUL +#define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE + +// F16 s390x +#define GGML_F16_STEP GGML_F32_STEP +#define GGML_F16_EPR GGML_F32_EPR + +static inline __vector float __lzs_f16cx4_load(const ggml_fp16_t * x) { + float tmp[4]; + + for (int i = 0; i < 4; i++) { + tmp[i] = GGML_FP16_TO_FP32(x[i]); + } + + return vec_xl(0, tmp); +} + +static inline void __lzs_f16cx4_store(ggml_fp16_t * x, __vector float y) { + float arr[4]; + + vec_xst(y, 0, arr); + + for (int i = 0; i < 4; i++) { + x[i] = GGML_FP32_TO_FP16(arr[i]); + } +} + +#define GGML_F16_VEC GGML_F32x4 +#define GGML_F16_VEC_ZERO GGML_F32x4_ZERO +#define GGML_F16_VEC_SET1 GGML_F32x4_SET1 +#define GGML_F16_VEC_LOAD(p, i) __lzs_f16cx4_load(p) +#define GGML_F16_VEC_STORE(p, r, i) __lzs_f16cx4_store(p, r[i]) +#define GGML_F16_VEC_FMA GGML_F32x4_FMA +#define GGML_F16_VEC_ADD GGML_F32x4_ADD +#define GGML_F16_VEC_MUL GGML_F32x4_MUL +#define GGML_F16_VEC_REDUCE GGML_F32x4_REDUCE + #endif // GGML_F32_ARR / GGML_F16_ARR // number of registers to use per step -#ifdef GGML_SIMD +// TODO: Remove logic bypass +#ifdef GGML_SIMD || (defined(__s390x__) && defined(__VEC__)) #define GGML_F32_ARR (GGML_F32_STEP/GGML_F32_EPR) #define GGML_F16_ARR (GGML_F16_STEP/GGML_F16_EPR) #endif @@ -1356,7 +1447,8 @@ static void ggml_vec_dot_f32(int n, float * restrict s, size_t bs, const float * UNUSED(by); UNUSED(bs); -#if defined(GGML_SIMD) +// TODO: Remove logic bypass +#if defined(GGML_SIMD) || (defined(__s390x__) && defined(__VEC__)) float sumf = 0.0f; const int np = (n & ~(GGML_F32_STEP - 1)); @@ -1469,7 +1561,8 @@ static void ggml_vec_dot_f16(int n, float * restrict s, size_t bs, ggml_fp16_t * ggml_float sumf = 0.0; -#if defined(GGML_SIMD) +// TODO: Remove logic bypass +#if defined(GGML_SIMD) || (defined(__s390x__) && defined(__VEC__)) const int np = (n & ~(GGML_F16_STEP - 1)); GGML_F16_VEC sum[GGML_F16_ARR] = { GGML_F16_VEC_ZERO }; @@ -1513,6 +1606,7 @@ inline static void ggml_vec_dot_f16_unroll(const int n, const int xs, float * re x[i] = (ggml_fp16_t *) ((char *) xv + i*xs); } +// TODO: Fix problematic (__vector float *) + (__vector float *) #if defined(GGML_SIMD) const int np = (n & ~(GGML_F16_STEP - 1)); @@ -1558,7 +1652,8 @@ inline static void ggml_vec_dot_f16_unroll(const int n, const int xs, float * re } inline static void ggml_vec_mad_f32(const int n, float * restrict y, const float * restrict x, const float v) { -#if defined(GGML_SIMD) +// TODO: Remove logic bypass +#if defined(GGML_SIMD) || (defined(__s390x__) && defined(__VEC__)) const int np = (n & ~(GGML_F32_STEP - 1)); GGML_F32_VEC vx = GGML_F32_VEC_SET1(v); @@ -1589,7 +1684,8 @@ inline static void ggml_vec_mad_f32(const int n, float * restrict y, const float } inline static void ggml_vec_mad_f16(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v) { -#if defined(GGML_SIMD) +// TODO: Remove logic bypass +#if defined(GGML_SIMD) || (defined(__s390x__) && defined(__VEC__)) const int np = (n & ~(GGML_F16_STEP - 1)); GGML_F16_VEC vx = GGML_F16_VEC_SET1(v); @@ -1630,7 +1726,8 @@ inline static void ggml_vec_mad_f32_unroll(const int n, const int xs, const int v[i] = (const float *) ((const char *) vv + i*vs); } -#if defined(GGML_SIMD) +// TODO: Remove logic bypass +#if defined(GGML_SIMD) || (defined(__s390x__) && defined(__VEC__)) const int np = (n & ~(GGML_F32_STEP - 1)); GGML_F32_VEC vx[GGML_VEC_MAD_UNROLL]; @@ -1675,7 +1772,8 @@ inline static void ggml_vec_mad_f32_unroll(const int n, const int xs, const int inline static void ggml_vec_scale_f32(const int n, float * y, const float v) { #if defined(GGML_USE_ACCELERATE) vDSP_vsmul(y, 1, &v, y, 1, n); -#elif defined(GGML_SIMD) +// TODO: Remove logic bypass +#elif defined(GGML_SIMD) || (defined(__s390x__) && defined(__VEC__)) const int np = (n & ~(GGML_F32_STEP - 1)); GGML_F32_VEC vx = GGML_F32_VEC_SET1(v); @@ -1704,7 +1802,8 @@ inline static void ggml_vec_scale_f32(const int n, float * y, const float v) { } inline static void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float v) { -#if defined(GGML_SIMD) +// TODO: Remove logic bypass +#if defined(GGML_SIMD) || (defined(__s390x__) && defined(__VEC__)) const int np = (n & ~(GGML_F16_STEP - 1)); GGML_F16_VEC vx = GGML_F16_VEC_SET1(v); From 32c1e11dd279c48dfd343414eaffb74fd2312324 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Wed, 1 Jan 2025 14:37:03 +0800 Subject: [PATCH 03/56] ggml: fix missing escape character in GGML_F32x4_REDUCE Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index 7a9ea95a5912a..5f17891912bf3 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -1250,11 +1250,11 @@ static inline void __lsx_f16x4_store(ggml_fp16_t * x, __m128 y) { for (int i = 0; i < offset; ++i) { \ sum[i] = vec_add(sum[i], sum[offset + i]); \ } \ - offset >>= 1; + offset >>= 1; \ for (int i = 0; i < offset; ++i) { \ sum[i] = vec_add(sum[i], sum[offset + i]); \ } \ - offset >>= 1; + offset >>= 1; \ for (int i = 0; i < offset; ++i) { \ sum[i] = vec_add(sum[i], sum[offset + i]); \ } \ From 518faffe72dd31a85626fa7562f10f6f39e819cf Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Wed, 1 Jan 2025 14:39:20 +0800 Subject: [PATCH 04/56] ggml: add temporary patch for GGML_F32_ARR and GGML_F16_ARR Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index 5f17891912bf3..efddafdd9a342 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -1308,6 +1308,9 @@ static inline void __lzs_f16cx4_store(ggml_fp16_t * x, __vector float y) { #define GGML_F16_VEC_MUL GGML_F32x4_MUL #define GGML_F16_VEC_REDUCE GGML_F32x4_REDUCE +#define GGML_F32_ARR (GGML_F32_STEP/GGML_F32_EPR) +#define GGML_F16_ARR (GGML_F16_STEP/GGML_F16_EPR) + #endif // GGML_F32_ARR / GGML_F16_ARR From b3779689a595be5327d90b1bdf8566556a164e09 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Wed, 1 Jan 2025 15:21:56 +0800 Subject: [PATCH 05/56] ggml: fix s390x GGML_F32x4_REDUCE Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index efddafdd9a342..8098b2d7d08cf 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -1248,15 +1248,15 @@ static inline void __lsx_f16x4_store(ggml_fp16_t * x, __m128 y) { { \ int offset = GGML_F32_ARR >> 1; \ for (int i = 0; i < offset; ++i) { \ - sum[i] = vec_add(sum[i], sum[offset + i]); \ + x[i] = vec_add(x[i], x[offset + i]); \ } \ offset >>= 1; \ for (int i = 0; i < offset; ++i) { \ - sum[i] = vec_add(sum[i], sum[offset + i]); \ + x[i] = vec_add(x[i], x[offset + i]); \ } \ offset >>= 1; \ for (int i = 0; i < offset; ++i) { \ - sum[i] = vec_add(sum[i], sum[offset + i]); \ + x[i] = vec_add(x[i], x[offset + i]); \ } \ res = vec_extract(x[0], 0) + \ vec_extract(x[0], 1) + \ @@ -1308,6 +1308,7 @@ static inline void __lzs_f16cx4_store(ggml_fp16_t * x, __vector float y) { #define GGML_F16_VEC_MUL GGML_F32x4_MUL #define GGML_F16_VEC_REDUCE GGML_F32x4_REDUCE +// TODO: Remove once SIMD is activated #define GGML_F32_ARR (GGML_F32_STEP/GGML_F32_EPR) #define GGML_F16_ARR (GGML_F16_STEP/GGML_F16_EPR) @@ -1315,8 +1316,7 @@ static inline void __lzs_f16cx4_store(ggml_fp16_t * x, __vector float y) { // GGML_F32_ARR / GGML_F16_ARR // number of registers to use per step -// TODO: Remove logic bypass -#ifdef GGML_SIMD || (defined(__s390x__) && defined(__VEC__)) +#ifdef GGML_SIMD #define GGML_F32_ARR (GGML_F32_STEP/GGML_F32_EPR) #define GGML_F16_ARR (GGML_F16_STEP/GGML_F16_EPR) #endif From 2dd768e9f8910358c4eb1b75b0d428ff9394b8ba Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Wed, 1 Jan 2025 15:45:44 +0800 Subject: [PATCH 06/56] ggml: full SIMD activation for F32,F16 s390x Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu.c | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index 8098b2d7d08cf..8898b0bcaad85 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -1228,8 +1228,7 @@ static inline void __lsx_f16x4_store(ggml_fp16_t * x, __m128 y) { #define vec_add(a, b) ((a) + (b)) #define vec_mul(a, b) ((a) * (b)) -// TODO: Activate this macro -//#define GGML_SIMD +#define GGML_SIMD // F32 s390x @@ -1308,10 +1307,6 @@ static inline void __lzs_f16cx4_store(ggml_fp16_t * x, __vector float y) { #define GGML_F16_VEC_MUL GGML_F32x4_MUL #define GGML_F16_VEC_REDUCE GGML_F32x4_REDUCE -// TODO: Remove once SIMD is activated -#define GGML_F32_ARR (GGML_F32_STEP/GGML_F32_EPR) -#define GGML_F16_ARR (GGML_F16_STEP/GGML_F16_EPR) - #endif // GGML_F32_ARR / GGML_F16_ARR @@ -1450,8 +1445,7 @@ static void ggml_vec_dot_f32(int n, float * restrict s, size_t bs, const float * UNUSED(by); UNUSED(bs); -// TODO: Remove logic bypass -#if defined(GGML_SIMD) || (defined(__s390x__) && defined(__VEC__)) +#if defined(GGML_SIMD) float sumf = 0.0f; const int np = (n & ~(GGML_F32_STEP - 1)); @@ -1564,8 +1558,7 @@ static void ggml_vec_dot_f16(int n, float * restrict s, size_t bs, ggml_fp16_t * ggml_float sumf = 0.0; -// TODO: Remove logic bypass -#if defined(GGML_SIMD) || (defined(__s390x__) && defined(__VEC__)) +#if defined(GGML_SIMD) const int np = (n & ~(GGML_F16_STEP - 1)); GGML_F16_VEC sum[GGML_F16_ARR] = { GGML_F16_VEC_ZERO }; @@ -1609,7 +1602,6 @@ inline static void ggml_vec_dot_f16_unroll(const int n, const int xs, float * re x[i] = (ggml_fp16_t *) ((char *) xv + i*xs); } -// TODO: Fix problematic (__vector float *) + (__vector float *) #if defined(GGML_SIMD) const int np = (n & ~(GGML_F16_STEP - 1)); @@ -1655,8 +1647,7 @@ inline static void ggml_vec_dot_f16_unroll(const int n, const int xs, float * re } inline static void ggml_vec_mad_f32(const int n, float * restrict y, const float * restrict x, const float v) { -// TODO: Remove logic bypass -#if defined(GGML_SIMD) || (defined(__s390x__) && defined(__VEC__)) +#if defined(GGML_SIMD) const int np = (n & ~(GGML_F32_STEP - 1)); GGML_F32_VEC vx = GGML_F32_VEC_SET1(v); @@ -1687,8 +1678,7 @@ inline static void ggml_vec_mad_f32(const int n, float * restrict y, const float } inline static void ggml_vec_mad_f16(const int n, ggml_fp16_t * restrict y, const ggml_fp16_t * restrict x, const float v) { -// TODO: Remove logic bypass -#if defined(GGML_SIMD) || (defined(__s390x__) && defined(__VEC__)) +#if defined(GGML_SIMD) const int np = (n & ~(GGML_F16_STEP - 1)); GGML_F16_VEC vx = GGML_F16_VEC_SET1(v); @@ -1729,8 +1719,7 @@ inline static void ggml_vec_mad_f32_unroll(const int n, const int xs, const int v[i] = (const float *) ((const char *) vv + i*vs); } -// TODO: Remove logic bypass -#if defined(GGML_SIMD) || (defined(__s390x__) && defined(__VEC__)) +#if defined(GGML_SIMD) const int np = (n & ~(GGML_F32_STEP - 1)); GGML_F32_VEC vx[GGML_VEC_MAD_UNROLL]; @@ -1775,8 +1764,7 @@ inline static void ggml_vec_mad_f32_unroll(const int n, const int xs, const int inline static void ggml_vec_scale_f32(const int n, float * y, const float v) { #if defined(GGML_USE_ACCELERATE) vDSP_vsmul(y, 1, &v, y, 1, n); -// TODO: Remove logic bypass -#elif defined(GGML_SIMD) || (defined(__s390x__) && defined(__VEC__)) +#elif defined(GGML_SIMD) const int np = (n & ~(GGML_F32_STEP - 1)); GGML_F32_VEC vx = GGML_F32_VEC_SET1(v); @@ -1805,8 +1793,7 @@ inline static void ggml_vec_scale_f32(const int n, float * y, const float v) { } inline static void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float v) { -// TODO: Remove logic bypass -#if defined(GGML_SIMD) || (defined(__s390x__) && defined(__VEC__)) +#if defined(GGML_SIMD) const int np = (n & ~(GGML_F16_STEP - 1)); GGML_F16_VEC vx = GGML_F16_VEC_SET1(v); From 0fdbc726420e9d70b2f45d8ffdcf0e0dd6e38482 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Wed, 1 Jan 2025 17:21:02 +0800 Subject: [PATCH 07/56] ggml: add option to disable s390x VXE/VXE2 Signed-off-by: Aaron Teo --- ggml/CMakeLists.txt | 1 + ggml/src/ggml-cpu/CMakeLists.txt | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index e33d974827cbe..ad689369d94e0 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -120,6 +120,7 @@ endif() option(GGML_LASX "ggml: enable lasx" ON) option(GGML_LSX "ggml: enable lsx" ON) option(GGML_RVV "ggml: enable rvv" ON) +option(GGML_VXE "ggml: enable vxe" ON) option(GGML_CPU_ALL_VARIANTS "ggml: build all variants of the CPU backend (requires GGML_BACKEND_DL)" OFF) set(GGML_CPU_ARM_ARCH "" CACHE STRING "ggml: CPU architecture for ARM") diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index 8658513b2dd0f..be27ec8c415cd 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -303,15 +303,18 @@ function(ggml_add_cpu_backend_variant_impl tag_name) if (GGML_LSX) list(APPEND ARCH_FLAGS -mlsx) endif() - elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "s390x") - message(STATUS "s390x detected") - - list(APPEND ARCH_FLAGS -march=z15 -mtune=z15 -mvx -mzvector) elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "riscv64") message(STATUS "RISC-V detected") if (GGML_RVV) list(APPEND ARCH_FLAGS -march=rv64gcv -mabi=lp64d) endif() + elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "s390x") + message(STATUS "s390x detected") + + list(APPEND ARCH_FLAGS -march=z15 -mtune=z15) + if (GGML_VXE) + list(APPEND ARCH_FLAGS -mvx -mzvector) + endif() else() message(STATUS "Unknown architecture") endif() From a44fba25602d6482d8c56b2960615188f426b66f Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Wed, 1 Jan 2025 17:54:15 +0800 Subject: [PATCH 08/56] ggml: change vecintrin.h include to ggml-cpu-impl * add __VXE__ and __VXE2__ macros Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-impl.h | 13 +++++++++++++ ggml/src/ggml-cpu/ggml-cpu.c | 8 ++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-impl.h b/ggml/src/ggml-cpu/ggml-cpu-impl.h index d71076ad12b1f..f2ddd3e09d249 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-impl.h +++ b/ggml/src/ggml-cpu/ggml-cpu-impl.h @@ -59,6 +59,15 @@ struct ggml_compute_params { #endif #endif +#if defined(__s390x__) && defined(__VEC__) +#ifndef __VXE__ +#define __VXE__ +#endif +#ifndef __VXE2__ +#define __VXE2__ +#endif +#endif + #if defined(__ARM_FEATURE_SVE) #include #include @@ -359,6 +368,10 @@ inline static int32x4_t ggml_vdotq_s32(int32x4_t acc, int8x16_t a, int8x16_t b) #endif #endif +#if defined(__VXE__) || defined(__VXE2__) +#include +#endif + #if defined(__loongarch_asx) typedef union { diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index 8898b0bcaad85..5e93ead0873ce 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -87,10 +87,6 @@ #include #endif -#if defined(__s390x__) && defined(__VEC__) -#include -#endif - // floating point type used to accumulate sums typedef double ggml_float; @@ -242,7 +238,7 @@ typedef pthread_t ggml_thread_t; #else #if defined(__POWER9_VECTOR__) #define CACHE_LINE_SIZE 128 -#elif defined(__s390x__) && defined(__VEC__) +#elif defined(__VXE__) || defined(__VXE2__) #define CACHE_LINE_SIZE 256 #else #define CACHE_LINE_SIZE 64 @@ -1224,7 +1220,7 @@ static inline void __lsx_f16x4_store(ggml_fp16_t * x, __m128 y) { #define GGML_F16_VEC_MUL GGML_F32Cx4_MUL #define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE -#elif defined(__s390x__) && defined(__VEC__) +#elif defined(__VXE__) || defined(__VXE2__) #define vec_add(a, b) ((a) + (b)) #define vec_mul(a, b) ((a) * (b)) From 77696c982c37657fd15555640910c6a84a616b11 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Thu, 2 Jan 2025 21:04:13 +0800 Subject: [PATCH 09/56] cmake: add s390x target detection for VX/VXE/VXE2 Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/CMakeLists.txt | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index be27ec8c415cd..aeb52843dcfe9 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -310,8 +310,21 @@ function(ggml_add_cpu_backend_variant_impl tag_name) endif() elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "s390x") message(STATUS "s390x detected") + execute_process(COMMAND bash -c "grep -Pom 1 'machine = \\K([0-9]+)' /proc/cpuinfo" OUTPUT_VARIABLE S390X_M) + + # TODO: Separation to determine activation of VX/VXE/VXE2 + if (${S390X_M} MATCHES "8561|8562") + message(STATUS "z15 target") + list(APPEND ARCH_FLAGS -march=z15 -mtune=z15) + elseif (${S390X_M} MATCHES "3931") + message(STATUS "z16 target") + list(APPEND ARCH_FLAGS -march=z16 -mtune=z16) + else() + message(STATUS "Unknown target") + message(WARNING "Unknown target. If you are compiling for z15 and earlier, you might have to add -DGGML_VXE=OFF.") + list(APPEND ARCH_FLAGS -march=native -mtune=native) + endif() - list(APPEND ARCH_FLAGS -march=z15 -mtune=z15) if (GGML_VXE) list(APPEND ARCH_FLAGS -mvx -mzvector) endif() From 47ca04746b6e51c0a22130e264a53152c5e5d9e7 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Sat, 4 Jan 2025 15:30:47 +0800 Subject: [PATCH 10/56] ggml: move s390x vector intrinsics to ggml-cpu-impl.h Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-impl.h | 24 ++++++++++++++++++++++++ ggml/src/ggml-cpu/ggml-cpu.c | 2 -- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-impl.h b/ggml/src/ggml-cpu/ggml-cpu-impl.h index f2ddd3e09d249..d14fbbe2eadc6 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-impl.h +++ b/ggml/src/ggml-cpu/ggml-cpu-impl.h @@ -370,6 +370,30 @@ inline static int32x4_t ggml_vdotq_s32(int32x4_t acc, int8x16_t a, int8x16_t b) #if defined(__VXE__) || defined(__VXE2__) #include + +#define vec_neg(a) (-(a)) // Vector Negate +#define vec_add(a, b) ((a) + (b)) // Vector Add +#define vec_sub(a, b) ((a) - (b)) // Vector Subtract +#define vec_mul(a, b) ((a) * (b)) // Vector Multiply +#define vec_div(a, b) ((a) / (b)) // Vector Divide +#define vec_sl(a, b) ((a) << (b)) // Vector Shift Left +#define vec_sra(a, b) ((a) >> (b)) // Vector Shift Right +#define vec_sr(a, b) ((a) >> (b)) // Vector Shift Right Algebraic +#define vec_slo(a, b) vec_slb(a, (b) << 64) // Vector Shift Left by Octet +#define vec_sro(a, b) vec_srb(a, (b) << 64) // Vector Shift Right by Octet + +#ifndef vec_and +#define vec_and(a, b) ((a) & (b)) // Vector AND +#endif + +#ifndef vec_or +#define vec_or(a, b) ((a) | (b)) // Vector OR +#endif + +#ifndef vec_xor +#define vec_xor(a, b) ((a) ^ (b)) // Vector XOR +#endif + #endif #if defined(__loongarch_asx) diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index 5e93ead0873ce..983708de995d2 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -1221,8 +1221,6 @@ static inline void __lsx_f16x4_store(ggml_fp16_t * x, __m128 y) { #define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE #elif defined(__VXE__) || defined(__VXE2__) -#define vec_add(a, b) ((a) + (b)) -#define vec_mul(a, b) ((a) * (b)) #define GGML_SIMD From 2d061928b9e3d24a6bc2b106ee69863acb485f1c Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Sat, 4 Jan 2025 19:57:35 +0800 Subject: [PATCH 11/56] ggml: s390x Q8_0 SIMD Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 55 +++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 8e14722667abb..f4e9efcd858ae 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -3686,6 +3686,61 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * r } sumf = hsum_float_8(acc); +#elif defined(__VXE__) || defined(__VXE2__) + __vector float sumv0 = vec_splats(0.0f); + __vector float sumv1 = vec_splats(0.0f); + + for (; ib + 1 < nb; ib += 2) { + const block_q8_0 * restrict x0 = &x[ib + 0]; + const block_q8_0 * restrict x1 = &x[ib + 1]; + const block_q8_0 * restrict y0 = &y[ib + 0]; + const block_q8_0 * restrict y1 = &y[ib + 1]; + + const float d0 = GGML_FP16_TO_FP32(x0->d) * GGML_FP16_TO_FP32(y0->d); + const float d1 = GGML_FP16_TO_FP32(x1->d) * GGML_FP16_TO_FP32(y1->d); + + // Load x0 and x1, low and high + const __vector int8_t x0_l = vec_xl(0 , x0->qs); + const __vector int8_t x0_h = vec_xl(QK8_0/2, x0->qs); + const __vector int8_t x1_l = vec_xl(0 , x1->qs); + const __vector int8_t x1_h = vec_xl(QK8_0/2, x1->qs); + + // Load y0 and y1, low and high + const __vector int8_t y0_l = vec_xl(0 , y0->qs); + const __vector int8_t y0_h = vec_xl(QK8_0/2, y0->qs); + const __vector int8_t y1_l = vec_xl(0 , y1->qs); + const __vector int8_t y1_h = vec_xl(QK8_0/2, y1->qs); + + const __vector int16_t xy0_lo = vec_mulo(x0_l, y0_l); + const __vector int16_t xy0_le = vec_mule(x0_l, y0_l); + const __vector int16_t xy0_ho = vec_mulo(x0_h, y0_h); + const __vector int16_t xy0_he = vec_mule(x0_h, y0_h); + + const __vector int16_t xy1_lo = vec_mulo(x1_l, y1_l); + const __vector int16_t xy1_le = vec_mule(x1_l, y1_l); + const __vector int16_t xy1_ho = vec_mulo(x1_h, y1_h); + const __vector int16_t xy1_he = vec_mule(x1_h, y1_h); + + __vector int16_t xy0_ = xy0_lo + xy0_le + xy0_ho + xy0_he; + __vector int16_t xy1_ = xy1_lo + xy1_le + xy1_ho + xy1_he; + + // Extend xy0_ and xy1_ from int16_t to int32_t + xy0_ += vec_reve(xy0_); + xy1_ += vec_reve(xy1_); + + // Unpack left-half to become int16_t and convert to float + const __vector float xy0 = vec_float(vec_unpackh(xy0_)); + const __vector float xy1 = vec_float(vec_unpackh(xy1_)); + + const __vector float v_d0 = { d0, d0, d0, d0 }; + const __vector float v_d1 = { d1, d1, d1, d1 }; + + sumv0 = vec_madd(xy0, v_d0, sumv0); + sumv1 = vec_madd(xy1, v_d1, sumv1); + } + + sumf = sumv0[0] + sumv0[1] + sumv0[2] + sumv0[3] + + sumv1[0] + sumv1[1] + sumv1[2] + sumv1[3]; #endif for (; ib < nb; ++ib) { int sumi = 0; From 33ea1d0366cb742bcf3938e26f0479ab89ffe700 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Sat, 4 Jan 2025 20:24:46 +0800 Subject: [PATCH 12/56] ggml: correct documentation for Q8_0 Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index f4e9efcd858ae..8122e266e461b 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -3724,7 +3724,7 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * r __vector int16_t xy0_ = xy0_lo + xy0_le + xy0_ho + xy0_he; __vector int16_t xy1_ = xy1_lo + xy1_le + xy1_ho + xy1_he; - // Extend xy0_ and xy1_ from int16_t to int32_t + // Fill remaining empty vector spaces xy0_ += vec_reve(xy0_); xy1_ += vec_reve(xy1_); From 82e045d8900072486b85e56d8b003d7069ab879a Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Sat, 4 Jan 2025 20:46:16 +0800 Subject: [PATCH 13/56] ggml: s390x reduce code complexity Q8_0 Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 63 ++++++++--------------------- 1 file changed, 16 insertions(+), 47 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 8122e266e461b..8a4642a1b7633 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -3687,60 +3687,29 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * r sumf = hsum_float_8(acc); #elif defined(__VXE__) || defined(__VXE2__) - __vector float sumv0 = vec_splats(0.0f); - __vector float sumv1 = vec_splats(0.0f); + __vector float acc = vec_splats(0.0f); - for (; ib + 1 < nb; ib += 2) { - const block_q8_0 * restrict x0 = &x[ib + 0]; - const block_q8_0 * restrict x1 = &x[ib + 1]; - const block_q8_0 * restrict y0 = &y[ib + 0]; - const block_q8_0 * restrict y1 = &y[ib + 1]; - - const float d0 = GGML_FP16_TO_FP32(x0->d) * GGML_FP16_TO_FP32(y0->d); - const float d1 = GGML_FP16_TO_FP32(x1->d) * GGML_FP16_TO_FP32(y1->d); - - // Load x0 and x1, low and high - const __vector int8_t x0_l = vec_xl(0 , x0->qs); - const __vector int8_t x0_h = vec_xl(QK8_0/2, x0->qs); - const __vector int8_t x1_l = vec_xl(0 , x1->qs); - const __vector int8_t x1_h = vec_xl(QK8_0/2, x1->qs); - - // Load y0 and y1, low and high - const __vector int8_t y0_l = vec_xl(0 , y0->qs); - const __vector int8_t y0_h = vec_xl(QK8_0/2, y0->qs); - const __vector int8_t y1_l = vec_xl(0 , y1->qs); - const __vector int8_t y1_h = vec_xl(QK8_0/2, y1->qs); - - const __vector int16_t xy0_lo = vec_mulo(x0_l, y0_l); - const __vector int16_t xy0_le = vec_mule(x0_l, y0_l); - const __vector int16_t xy0_ho = vec_mulo(x0_h, y0_h); - const __vector int16_t xy0_he = vec_mule(x0_h, y0_h); - - const __vector int16_t xy1_lo = vec_mulo(x1_l, y1_l); - const __vector int16_t xy1_le = vec_mule(x1_l, y1_l); - const __vector int16_t xy1_ho = vec_mulo(x1_h, y1_h); - const __vector int16_t xy1_he = vec_mule(x1_h, y1_h); - - __vector int16_t xy0_ = xy0_lo + xy0_le + xy0_ho + xy0_he; - __vector int16_t xy1_ = xy1_lo + xy1_le + xy1_ho + xy1_he; + for (; ib < nb; ++ib) { + const __vector int8_t v_xl = vec_xl(0 , x[ib].qs); + const __vector int8_t v_xh = vec_xl(QK8_0/2, x[ib].qs); + const __vector int8_t v_yl = vec_xl(0 , y[ib].qs); + const __vector int8_t v_yh = vec_xl(QK8_0/2, y[ib].qs); - // Fill remaining empty vector spaces - xy0_ += vec_reve(xy0_); - xy1_ += vec_reve(xy1_); + const __vector int16_t v_xylo = vec_mulo(v_xl, v_yl); + const __vector int16_t v_xyle = vec_mule(v_xl, v_yl); + const __vector int16_t v_xyho = vec_mulo(v_xh, v_yh); + const __vector int16_t v_xyhe = vec_mule(v_xh, v_yh); - // Unpack left-half to become int16_t and convert to float - const __vector float xy0 = vec_float(vec_unpackh(xy0_)); - const __vector float xy1 = vec_float(vec_unpackh(xy1_)); + __vector int16_t v_xy_ = v_xylo + v_xyle + v_xyho + v_xyhe; + v_xy_ += vec_reve(v_xy_); - const __vector float v_d0 = { d0, d0, d0, d0 }; - const __vector float v_d1 = { d1, d1, d1, d1 }; + const __vector float v_xy = vec_float(vec_unpackh(xy_)); + const __vector float v_d = vec_splats(GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d)); - sumv0 = vec_madd(xy0, v_d0, sumv0); - sumv1 = vec_madd(xy1, v_d1, sumv1); + acc = vec_madd(v_xy, v_d, acc); } - sumf = sumv0[0] + sumv0[1] + sumv0[2] + sumv0[3] + - sumv1[0] + sumv1[1] + sumv1[2] + sumv1[3]; + sumf = acc[0] + acc[1] + acc[2] + acc[3]; #endif for (; ib < nb; ++ib) { int sumi = 0; From 261689d4ea5e6a5ab42a46e6263f50bdc06c726f Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Sat, 4 Jan 2025 20:47:37 +0800 Subject: [PATCH 14/56] ggml: s390x bugfix typo Q8_0 Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 8a4642a1b7633..7dd214d6d398b 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -3703,7 +3703,7 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * r __vector int16_t v_xy_ = v_xylo + v_xyle + v_xyho + v_xyhe; v_xy_ += vec_reve(v_xy_); - const __vector float v_xy = vec_float(vec_unpackh(xy_)); + const __vector float v_xy = vec_float(vec_unpackh(v_xy_)); const __vector float v_d = vec_splats(GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d)); acc = vec_madd(v_xy, v_d, acc); From 4212c464dfc736508782b6893bae950e1fd3d84f Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Sun, 5 Jan 2025 14:53:55 +0800 Subject: [PATCH 15/56] ggml: s390x SIMD activated for Q4_1 Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 31 +++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 7dd214d6d398b..7124f928c490c 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -2591,6 +2591,37 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r } sumf = hsum_float_8(acc) + summs; +#elif defined(__VXE__) || defined(__VXE2__) + float summs = 0; + __vector float acc = vec_splats(0.0f); + + const __vector uint8_t v_m = vec_splats((const uint8_t)0x0F); + + for (; ib < nb; ++ib) { + summs += GGML_FP16_TO_FP32(x[ib].m) * GGML_FP16_TO_FP32(y[ib].s); + + const __vector uint8_t v_x = vec_xl(0, x[ib].qs); + const __vector int8_t v_xl = (const __vector int8_t)(v_x & v_m); + const __vector int8_t v_xh = (const __vector int8_t)(v_x >> 4); + + const __vector int8_t v_yl = vec_xl(0 , y[ib].qs); + const __vector int8_t v_yh = vec_xl(QK8_1/2, y[ib].qs); + + const __vector int16_t xylo = vec_mulo(v_xl, v_yl); + const __vector int16_t xyle = vec_mule(v_xl, v_yl); + const __vector int16_t xyho = vec_mulo(v_xh, v_yh); + const __vector int16_t xyhe = vec_mule(v_xh, v_yh); + + __vector int16_t v_xy_ = xylo + xyle + xyho + xyhe; + v_xy_ += vec_reve(v_xy_); + + const __vector float v_xy = vec_float(vec_unpackh(v_xy_)); + const __vector float v_d = vec_splats(GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d)); + + acc = vec_madd(v_xy, v_d, acc); + } + + sumf = acc[0] + acc[1] + acc[2] + acc[3] + summs; #endif for (; ib < nb; ++ib) { int sumi0 = 0; From 44402b766ee1cb792ba8138d5abbe06facf452e1 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Sun, 5 Jan 2025 16:08:47 +0800 Subject: [PATCH 16/56] ggml: s390x inline vec_reve Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 7124f928c490c..12e85db0546c1 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -2612,8 +2612,7 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r const __vector int16_t xyho = vec_mulo(v_xh, v_yh); const __vector int16_t xyhe = vec_mule(v_xh, v_yh); - __vector int16_t v_xy_ = xylo + xyle + xyho + xyhe; - v_xy_ += vec_reve(v_xy_); + __vector int16_t v_xy_ = xylo + xyle + xyho + xyhe; v_xy_ += vec_reve(v_xy_); const __vector float v_xy = vec_float(vec_unpackh(v_xy_)); const __vector float v_d = vec_splats(GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d)); @@ -3731,8 +3730,7 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * r const __vector int16_t v_xyho = vec_mulo(v_xh, v_yh); const __vector int16_t v_xyhe = vec_mule(v_xh, v_yh); - __vector int16_t v_xy_ = v_xylo + v_xyle + v_xyho + v_xyhe; - v_xy_ += vec_reve(v_xy_); + __vector int16_t v_xy_ = v_xylo + v_xyle + v_xyho + v_xyhe; v_xy_ += vec_reve(v_xy_); const __vector float v_xy = vec_float(vec_unpackh(v_xy_)); const __vector float v_d = vec_splats(GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d)); From 68760a89767ae3a51695c868e1d285bfcd1acd7e Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Sun, 5 Jan 2025 19:06:16 +0800 Subject: [PATCH 17/56] ggml: s390x SIMD activation for Q4_0 Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 31 +++++++++++++++++++++++++++++ test.py | 11 ++++++++++ 2 files changed, 42 insertions(+) create mode 100644 test.py diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 12e85db0546c1..204304e8313ba 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -2298,6 +2298,37 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * r } sumf = hsum_float_4x4(acc_0, acc_1, acc_2, acc_3); +#elif defined(__VXE__) || defined(__VXE2__) + __vector float acc = vec_splats(0.0f); + + const __vector uint8_t v_m = vec_splats((const uint8_t)0x0F); + const __vector int8_t v_s = vec_splats( (const int8_t)0x08); + + for (; ib < nb; ++ib) { + const __vector uint8_t v_x = vec_xl(0, x[ib].qs); + const __vector int8_t v_xl = (const __vector int8_t)(v_x & v_m); + const __vector int8_t v_xh = (const __vector int8_t)(v_x >> 4); + + const __vector int8_t v_xls = vec_sub(v_xl, v_s); + const __vector int8_t v_xhs = vec_sub(v_xh, v_s); + + const __vector int8_t v_yl = vec_xl(0 , y[ib].qs); + const __vector int8_t v_yh = vec_xl(QK8_0/2, y[ib].qs); + + const __vector int16_t v_xylso = vec_mulo(v_xls, v_yl); + const __vector int16_t v_xylse = vec_mule(v_xls, v_yl); + const __vector int16_t v_xyhso = vec_mulo(v_xhs, v_yh); + const __vector int16_t v_xyhse = vec_mule(v_xhs, v_yh); + + __vector int16_t v_xy_ = v_xylso + v_xylse + v_xyhso + v_xyhse; v_xy_ += vec_reve(v_xy_); + + const __vector float v_xy = vec_float(vec_unpackh(v_xy_)); + const __vector float v_d = vec_splats(GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d)); + + acc = vec_madd(v_xy, v_d, acc); + } + + sumf = acc[0] + acc[1] + acc[2] + acc[3]; #endif for (; ib < nb; ++ib) { int sumi0 = 0; diff --git a/test.py b/test.py new file mode 100644 index 0000000000000..022735976678d --- /dev/null +++ b/test.py @@ -0,0 +1,11 @@ +x = 5 +y = 8 +z = 0 + +for a in range(1, 10 + 1): + z = 0 + for num in range(a-2): + z = z + num ** 3 + x * num + num - y + + if z == 120: + print(a) From ecdf6f0e8a532172c0ce5e51e520eb78b41ebec5 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Tue, 7 Jan 2025 14:57:17 +0800 Subject: [PATCH 18/56] ggml: add VXE backend feature Signed-off-by: Aaron Teo --- ggml/include/ggml-cpu.h | 1 + ggml/src/ggml-cpu/ggml-cpu.c | 8 ++++++++ ggml/src/ggml-cpu/ggml-cpu.cpp | 3 +++ 3 files changed, 12 insertions(+) diff --git a/ggml/include/ggml-cpu.h b/ggml/include/ggml-cpu.h index 3aa71badb5fb0..8f17e57b00e86 100644 --- a/ggml/include/ggml-cpu.h +++ b/ggml/include/ggml-cpu.h @@ -98,6 +98,7 @@ extern "C" { // other GGML_BACKEND_API int ggml_cpu_has_riscv_v (void); GGML_BACKEND_API int ggml_cpu_has_vsx (void); + GGML_BACKEND_API int ggml_cpu_has_vxe (void); GGML_BACKEND_API int ggml_cpu_has_wasm_simd (void); GGML_BACKEND_API int ggml_cpu_has_llamafile (void); diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index 983708de995d2..aa89764e959ae 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -14123,6 +14123,14 @@ int ggml_cpu_has_vsx(void) { #endif } +int ggml_cpu_has_vxe(void) { +#if defined(__VXE__) || defined(__VXE2__) + return 1; +#else + return 0; +#endif +} + int ggml_cpu_has_neon(void) { #if defined(__ARM_ARCH) && defined(__ARM_NEON) return ggml_arm_arch_features.has_neon; diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp index f11399cc628ca..3a1f2a9ca851a 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.cpp +++ b/ggml/src/ggml-cpu/ggml-cpu.cpp @@ -538,6 +538,9 @@ static ggml_backend_feature * ggml_backend_cpu_get_features(ggml_backend_reg_t r if (ggml_cpu_has_vsx()) { features.push_back({ "VSX", "1" }); } + if (ggml_cpu_has_vxe()) { + features.push_back({ "VXE", "1" }); + } if (ggml_cpu_has_wasm_simd()) { features.push_back({ "WASM_SIMD", "1" }); } From fd993b283cbac4a12bff6789d37f03cf978a157c Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Tue, 7 Jan 2025 14:57:46 +0800 Subject: [PATCH 19/56] ggml: remove test.py Signed-off-by: Aaron Teo --- test.py | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 test.py diff --git a/test.py b/test.py deleted file mode 100644 index 022735976678d..0000000000000 --- a/test.py +++ /dev/null @@ -1,11 +0,0 @@ -x = 5 -y = 8 -z = 0 - -for a in range(1, 10 + 1): - z = 0 - for num in range(a-2): - z = z + num ** 3 + x * num + num - y - - if z == 120: - print(a) From 0f1e7a07cbf910db98b5530dc3fc6f91f5521a72 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Tue, 7 Jan 2025 18:16:49 +0800 Subject: [PATCH 20/56] ggml: s390x SIMD activation for quantize_row_q8_0 Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 32 +++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 204304e8313ba..51165f638108f 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -988,6 +988,38 @@ void quantize_row_q8_0(const float * restrict x, void * restrict vy, int64_t k) __lsx_vst(ni4, (__m128i *)(y[i].qs + 16), 0); } +#elif defined(__VXE__) || defined(__VXE2__) + for (int i = 0; i < nb; i++) { + __vector float srcv [8]; + __vector float asrcv[8]; + __vector float amaxv[8]; + + for (int j = 0; j < 8; j++) srcv[j] = vec_xl(0, x + i*32 + 4*j); + for (int j = 0; j < 8; j++) asrcv[j] = vec_abs(srcv[j]); + for (int j = 0; j < 4; j++) amaxv[2*j] = vec_max(asrcv[2*j], asrcv[2*j+1]); + for (int j = 0; j < 2; j++) amaxv[4*j] = vec_max(amaxv[4*j], amaxv[4*j+2]); + for (int j = 0; j < 1; j++) amaxv[8*j] = vec_max(amaxv[8*j], amaxv[8*j+4]); + + const float amax = MAX(MAX(vec_extract(amaxv[0], 0), + vec_extract(amaxv[0], 1)), + MAX(vec_extract(amaxv[0], 2), + vec_extract(amaxv[0], 3))); + + const float d = amax / ((1 << 7) - 1); + const float id = d ? 1.0f / d : 0.0f; + + y[i].d = GGML_FP32_TO_FP16(d); + + for (int j = 0; j < 8; j++) { + const __vector float v = vec_mul(srcv[j], vec_splats(id)); + const __vector int32_t vi = vec_signed(v); + + y[i].qs[4*j + 0] = vec_extract(vi, 0); + y[i].qs[4*j + 1] = vec_extract(vi, 1); + y[i].qs[4*j + 2] = vec_extract(vi, 2); + y[i].qs[4*j + 3] = vec_extract(vi, 3); + } + } #else GGML_UNUSED(nb); // scalar From cd707a7b138e225a3c05b25a49f7c8c503298fcc Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Wed, 8 Jan 2025 17:08:48 +0800 Subject: [PATCH 21/56] ggml: s390x SIMD activation for quantize_row_q8_1 Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 38 +++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 51165f638108f..71f2737bb03f3 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -1348,6 +1348,44 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int64_t k) __lsx_vst(ni0, (__m128i *)(y[i].qs + 0), 0); __lsx_vst(ni4, (__m128i *)(y[i].qs + 16), 0); } +#elif defined(__VXE__) || defined(__VXE2__) + for (int i = 0; i < nb; i++) { + __vector float srcv [8]; + __vector float asrcv[8]; + __vector float amaxv[8]; + + for (int j = 0; j < 8; j++) srcv[j] = vec_xl(0, x + i*32 + 4*j); + for (int j = 0; j < 8; j++) asrcv[j] = vec_abs(srcv[j]); + for (int j = 0; j < 4; j++) amaxv[2*j] = vec_max(asrcv[2*j], asrcv[2*j+1]); + for (int j = 0; j < 2; j++) amaxv[4*j] = vec_max(amaxv[4*j], amaxv[4*j+2]); + for (int j = 0; j < 1; j++) amaxv[8*j] = vec_max(amaxv[8*j], amaxv[8*j+4]); + + const float amax = MAX(MAX(vec_extract(amaxv[0], 0), + vec_extract(amaxv[0], 1)), + MAX(vec_extract(amaxv[0], 2), + vec_extract(amaxv[0], 3))); + + const float d = amax / ((1 << 7) - 1); + const float id = d ? 1.0f / d : 0.0f; + + y[i].d = GGML_FP32_TO_FP16(d); + + __vector int32_t acc = vec_splats(0); + + for (int j = 0; j < 8; j++) { + const __vector float v = vec_mul(srcv[j], vec_splats(id)); + const __vector int32_t vi = vec_signed(v); + + y[i].qs[4*j + 0] = vec_extract(vi, 0); + y[i].qs[4*j + 1] = vec_extract(vi, 1); + y[i].qs[4*j + 2] = vec_extract(vi, 2); + y[i].qs[4*j + 3] = vec_extract(vi, 3); + + acc = vec_add(acc, vi); + } + + y[i].s = GGML_FP32_TO_FP16(d * (acc[0] + acc[1] + acc[2] + acc[3])); + } #else GGML_UNUSED(nb); // scalar From e1f939fc6d9411b7dd6ac90af1859de25048aab9 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Tue, 14 Jan 2025 16:11:23 +0800 Subject: [PATCH 22/56] ggml: s390x SIMD activation for iq4_xs Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-impl.h | 64 +++++++++++++++++++++++++++++ ggml/src/ggml-cpu/ggml-cpu-quants.c | 42 +++++++++++++++++++ 2 files changed, 106 insertions(+) diff --git a/ggml/src/ggml-cpu/ggml-cpu-impl.h b/ggml/src/ggml-cpu/ggml-cpu-impl.h index d14fbbe2eadc6..7d2a4668d283f 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-impl.h +++ b/ggml/src/ggml-cpu/ggml-cpu-impl.h @@ -394,6 +394,70 @@ inline static int32x4_t ggml_vdotq_s32(int32x4_t acc, int8x16_t a, int8x16_t b) #define vec_xor(a, b) ((a) ^ (b)) // Vector XOR #endif +typedef int8_t int8x16_t __attribute__((vector_size(16))); +typedef int16_t int16x8_t __attribute__((vector_size(16))); +typedef int32_t int32x4_t __attribute__((vector_size(16))); + +typedef uint8_t uint8x16_t __attribute__((vector_size(16))); +typedef uint16_t uint16x8_t __attribute__((vector_size(16))); +typedef uint32_t uint32x4_t __attribute__((vector_size(16))); + +typedef struct ggml_uint8x2_t { + uint8x16_t val[2]; +} ggml_uint8x2_t; + +typedef struct ggml_int8x4_t { + int8x16_t val[4]; +} ggml_int8x4_t; + +inline static ggml_uint8x2_t ggml_vec_xl_x2(const uint8_t * ptr) { + ggml_uint8x2_t res; + + res.val[0] = vec_xl( 0, ptr); + res.val[1] = vec_xl(16, ptr); + + return res; +} + +inline static ggml_int8x4_t ggml_vec_xl_x4(const int8_t * ptr) { + ggml_int8x4_t res; + + res.val[0] = vec_xl( 0, ptr); + res.val[1] = vec_xl(16, ptr); + res.val[2] = vec_xl(32, ptr); + res.val[3] = vec_xl(48, ptr); + + return res; +} + +inline static int8x16_t ggml_vec_tbl(int8x16_t a, uint8x16_t b) { + int8x16_t res; + + res[ 0] = a[b[ 0]]; + res[ 1] = a[b[ 1]]; + res[ 2] = a[b[ 2]]; + res[ 3] = a[b[ 3]]; + res[ 4] = a[b[ 4]]; + res[ 5] = a[b[ 5]]; + res[ 6] = a[b[ 6]]; + res[ 7] = a[b[ 7]]; + res[ 8] = a[b[ 8]]; + res[ 9] = a[b[ 9]]; + res[10] = a[b[10]]; + res[11] = a[b[11]]; + res[12] = a[b[12]]; + res[13] = a[b[13]]; + res[14] = a[b[14]]; + res[15] = a[b[15]]; + + return res; +} + +inline static int32x4_t ggml_vec_dot(int32x4_t acc, int8x16_t a, int8x16_t b) { + const int16x8_t p = vec_mule(a, b) + vec_mulo(a, b); + return acc + (vec_unpackh(p) + vec_unpackl(p)); +} + #endif #if defined(__loongarch_asx) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 71f2737bb03f3..f2fb1fb258a26 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -10944,6 +10944,48 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * restrict s, size_t bs, const void * } *s = hsum_float_8(accum); +#elif defined(__VXE__) || defined(__VXE2__) + const int8x16_t v_k = vec_xl(0, kvalues_iq4nl); + const uint8x16_t v_m = vec_splat_u8(0x0F); + + ggml_uint8x2_t q4bits; + ggml_int8x4_t q4b, q8b; + int32x4_t prod_1, prod_2; + + float sumf = 0; + + for (int ibl = 0; ibl < nb; ++ibl) { + const uint8_t * q4 = x[ibl].qs; + const int8_t * q8 = y[ibl].qs; + + uint16_t h = x[ibl].scales_h; + + int sumi1 = 0, sumi2 = 0; + for (int ib = 0; ib < QK_K/64; ++ib) { + q4bits = ggml_vec_xl_x2(q4); q4 += 32; + q8b = ggml_vec_xl_x4(q8); q8 += 64; + + q4b.val[0] = ggml_vec_tbl(v_k, vec_and(q4bits.val[0], v_m)); + q4b.val[1] = ggml_vec_tbl(v_k, vec_sr(q4bits.val[0], 4)); + q4b.val[2] = ggml_vec_tbl(v_k, vec_and(q4bits.val[1], v_m)); + q4b.val[3] = ggml_vec_tbl(v_k, vec_sr(q4bits.val[1], 4)); + + prod_1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), q4b.val[0], q8b.val[0]), q4b.val[1], q8b.val[1]); + prod_2 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), q4b.val[2], q8b.val[2]), q4b.val[3], q8b.val[3]); + + int ls1 = ((x[ibl].scales_l[ib] & 0xF) | ((h << 4) & 0x30)) - 32; + int ls2 = ((x[ibl].scales_l[ib] >> 4) | ((h << 2) & 0x30)) - 32; + + h >>= 4; + + sumi1 += (prod_1[0] + prod_1[1] + prod_1[2] + prod_1[3]) * ls1; + sumi2 += (prod_2[0] + prod_2[1] + prod_2[2] + prod_2[3]) * ls1; + } + + sumf += GGML_FP16_TO_FP32(x[ibl].d) * y[ibl].d * (sumi1 + sumi2); + } + + *s = sumf; #else float sumf = 0; From 37a0a62f14de6edf79baa85c7f0993a70e7c662e Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Tue, 14 Jan 2025 16:17:05 +0800 Subject: [PATCH 23/56] ggml: bugfix iq4_xs Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index f2fb1fb258a26..1b768d434a87a 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -10979,7 +10979,7 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * restrict s, size_t bs, const void * h >>= 4; sumi1 += (prod_1[0] + prod_1[1] + prod_1[2] + prod_1[3]) * ls1; - sumi2 += (prod_2[0] + prod_2[1] + prod_2[2] + prod_2[3]) * ls1; + sumi2 += (prod_2[0] + prod_2[1] + prod_2[2] + prod_2[3]) * ls2; } sumf += GGML_FP16_TO_FP32(x[ibl].d) * y[ibl].d * (sumi1 + sumi2); From 8df026936aa8537532550a2516c7695ec90aca90 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Tue, 14 Jan 2025 18:39:51 +0800 Subject: [PATCH 24/56] ggml: s390x SIMD activation for iq4_nl Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-impl.h | 4 ++++ ggml/src/ggml-cpu/ggml-cpu-quants.c | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/ggml/src/ggml-cpu/ggml-cpu-impl.h b/ggml/src/ggml-cpu/ggml-cpu-impl.h index 7d2a4668d283f..10830ef4cd4d4 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-impl.h +++ b/ggml/src/ggml-cpu/ggml-cpu-impl.h @@ -394,6 +394,9 @@ inline static int32x4_t ggml_vdotq_s32(int32x4_t acc, int8x16_t a, int8x16_t b) #define vec_xor(a, b) ((a) ^ (b)) // Vector XOR #endif +typedef signed char char1x16_t __attribute__((vector_size(16))); +typedef unsigned char uchar1x16_t __attribute__((vector_size(16))); + typedef int8_t int8x16_t __attribute__((vector_size(16))); typedef int16_t int16x8_t __attribute__((vector_size(16))); typedef int32_t int32x4_t __attribute__((vector_size(16))); @@ -430,6 +433,7 @@ inline static ggml_int8x4_t ggml_vec_xl_x4(const int8_t * ptr) { return res; } +//! WARNING: Very slow. Do not use if possible. inline static int8x16_t ggml_vec_tbl(int8x16_t a, uint8x16_t b) { int8x16_t res; diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 1b768d434a87a..14adc480ce834 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -10630,6 +10630,24 @@ void ggml_vec_dot_iq4_nl_q8_0(int n, float * restrict s, size_t bs, const void * sumf = hsum_float_8(__lasx_xvfadd_s(accum1, accum2)); +#elif defined(__VXE__) || defined(__VXE2__) + const int8x16_t v_k = vec_xl(0, kvalues_iq4nl); + const uint8x16_t v_m = vec_splat_u8(0x0F); + + for (; ib < nb; ++ib) { + const uint8x16_t v_x = vec_xl(0, x[ib].qs); + int8x16_t v_xl = (int8x16_t)vec_and(v_x, v_m); + int8x16_t v_xh = (int8x16_t)vec_sr(v_x, 4); + + v_xl = vec_perm(v_k, v_k, (uchar1x16_t)v_xl); + v_xh = vec_perm(v_k, v_k, (uchar1x16_t)v_xh); + + const int8x16_t v_yl = vec_xl( 0, y[ib].qs); + const int8x16_t v_yh = vec_xl(QK8_0/2, y[ib].qs); + const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh); + + sumf += GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d) * (v_xy[0] + v_xy[1] + v_xy[2] + v_xy[3]); + } #endif for (; ib < nb; ++ib) { const float d = GGML_FP16_TO_FP32(y[ib].d)*GGML_FP16_TO_FP32(x[ib].d); From ee750c9c16f2088e693845ad9ca6fd285141f1c6 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Tue, 14 Jan 2025 18:46:56 +0800 Subject: [PATCH 25/56] ggml: add float, double, and long vector data type Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-impl.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ggml/src/ggml-cpu/ggml-cpu-impl.h b/ggml/src/ggml-cpu/ggml-cpu-impl.h index 10830ef4cd4d4..ae2c8c13118e3 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-impl.h +++ b/ggml/src/ggml-cpu/ggml-cpu-impl.h @@ -405,6 +405,12 @@ typedef uint8_t uint8x16_t __attribute__((vector_size(16))); typedef uint16_t uint16x8_t __attribute__((vector_size(16))); typedef uint32_t uint32x4_t __attribute__((vector_size(16))); +typedef float float32x4_t __attribute__((vector_size(16))); +typedef double double64x2_t __attribute((vector_size(16))); + +typedef signed long long long64x2_t __attribute((vector_size(16))); +typedef unsigned long long ulong64x2_t __attribute__((vector_size(16))); + typedef struct ggml_uint8x2_t { uint8x16_t val[2]; } ggml_uint8x2_t; From 2073291f436eb9c033d58d0850e04e00ff6306e8 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Tue, 14 Jan 2025 19:37:03 +0800 Subject: [PATCH 26/56] ggml: clean up iq4_xs SIMD Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 40 +++++++++++++++++------------ 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 14adc480ce834..96e389ec4334d 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -10966,38 +10966,46 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * restrict s, size_t bs, const void * const int8x16_t v_k = vec_xl(0, kvalues_iq4nl); const uint8x16_t v_m = vec_splat_u8(0x0F); - ggml_uint8x2_t q4bits; - ggml_int8x4_t q4b, q8b; - int32x4_t prod_1, prod_2; - float sumf = 0; for (int ibl = 0; ibl < nb; ++ibl) { - const uint8_t * q4 = x[ibl].qs; - const int8_t * q8 = y[ibl].qs; + const uint8_t restrict * q4 = x[ibl].qs; + const int8_t restrict * q8 = y[ibl].qs; uint16_t h = x[ibl].scales_h; int sumi1 = 0, sumi2 = 0; for (int ib = 0; ib < QK_K/64; ++ib) { - q4bits = ggml_vec_xl_x2(q4); q4 += 32; - q8b = ggml_vec_xl_x4(q8); q8 += 64; + const uint8x16_t v_x0 = vec_xl(0 , q4); + const uint8x16_t v_x1 = vec_xl(QK4_NL/2, q4); + q4 += 32; + + int8x16_t v_x0l = (int8x16_t)vec_and(v_x0, v_m); + int8x16_t v_x0h = (int8x16_t)vec_sr(v_x0, 4); + int8x16_t v_x1l = (int8x16_t)vec_and(v_x1, v_m); + int8x16_t v_x1h = (int8x16_t)vec_sr(v_x1, 4); - q4b.val[0] = ggml_vec_tbl(v_k, vec_and(q4bits.val[0], v_m)); - q4b.val[1] = ggml_vec_tbl(v_k, vec_sr(q4bits.val[0], 4)); - q4b.val[2] = ggml_vec_tbl(v_k, vec_and(q4bits.val[1], v_m)); - q4b.val[3] = ggml_vec_tbl(v_k, vec_sr(q4bits.val[1], 4)); + v_x0l = vec_perm(v_k, v_k, (uchar1x16_t)v_x0l); + v_x0h = vec_perm(v_k, v_k, (uchar1x16_t)v_x0h); + v_x1l = vec_perm(v_k, v_k, (uchar1x16_t)v_x1l); + v_x1h = vec_perm(v_k, v_k, (uchar1x16_t)v_x1h); + + const int8x16_t v_y0 = vec_xl( 0, q8); + const int8x16_t v_y1 = vec_xl(16, q8); + const int8x16_t v_y2 = vec_xl(32, q8); + const int8x16_t v_y3 = vec_xl(48, q8); + q8 += 64; - prod_1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), q4b.val[0], q8b.val[0]), q4b.val[1], q8b.val[1]); - prod_2 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), q4b.val[2], q8b.val[2]), q4b.val[3], q8b.val[3]); + int32x4_t vsumi0 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x0l, v_y0), v_x0h, v_y1); + int32x4_t vsumi1 = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_x1l, v_y2), v_x1h, v_y3); int ls1 = ((x[ibl].scales_l[ib] & 0xF) | ((h << 4) & 0x30)) - 32; int ls2 = ((x[ibl].scales_l[ib] >> 4) | ((h << 2) & 0x30)) - 32; h >>= 4; - sumi1 += (prod_1[0] + prod_1[1] + prod_1[2] + prod_1[3]) * ls1; - sumi2 += (prod_2[0] + prod_2[1] + prod_2[2] + prod_2[3]) * ls2; + sumi1 += (vsumi0[0] + vsumi0[1] + vsumi0[2] + vsumi0[3]) * ls1; + sumi2 += (vsumi1[0] + vsumi1[1] + vsumi1[2] + vsumi1[3]) * ls2; } sumf += GGML_FP16_TO_FP32(x[ibl].d) * y[ibl].d * (sumi1 + sumi2); From 0c6e6d6531f374d0e111b2f8dacb16055bb5aea4 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Tue, 14 Jan 2025 19:39:58 +0800 Subject: [PATCH 27/56] ggml: fix improper use of restrict keyword Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 96e389ec4334d..c9b3faa56b22c 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -10969,8 +10969,8 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * restrict s, size_t bs, const void * float sumf = 0; for (int ibl = 0; ibl < nb; ++ibl) { - const uint8_t restrict * q4 = x[ibl].qs; - const int8_t restrict * q8 = y[ibl].qs; + const uint8_t * restrict q4 = x[ibl].qs; + const int8_t * restrict q8 = y[ibl].qs; uint16_t h = x[ibl].scales_h; From 109be7ffdb8aac5ebab13f6f821a94de3f1d802f Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Tue, 14 Jan 2025 19:42:44 +0800 Subject: [PATCH 28/56] ggml: update warning message for ggml_vec_tbl Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-impl.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-impl.h b/ggml/src/ggml-cpu/ggml-cpu-impl.h index ae2c8c13118e3..617fd7133147e 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-impl.h +++ b/ggml/src/ggml-cpu/ggml-cpu-impl.h @@ -439,7 +439,10 @@ inline static ggml_int8x4_t ggml_vec_xl_x4(const int8_t * ptr) { return res; } -//! WARNING: Very slow. Do not use if possible. +/* + ! WARNING: Very slow. Use vec_perm if possible. Refer to iq4_xs + ! or iq4_nl for example implementation. +*/ inline static int8x16_t ggml_vec_tbl(int8x16_t a, uint8x16_t b) { int8x16_t res; From ed6487c813d89032a5cb652407c74dccb9a71ca5 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Fri, 17 Jan 2025 18:16:08 +0800 Subject: [PATCH 29/56] ggml: untested implementation of ggml_vec_dot_iq2_xxs_q8_K Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 52 ++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index c9b3faa56b22c..039963161fea4 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -7865,7 +7865,57 @@ void ggml_vec_dot_iq2_xxs_q8_K(int n, float * restrict s, size_t bs, const void } *s = 0.125f * hsum_float_8(accumf); - +//#elif defined(__VXE__) || defined(__VXE2__) +// const uint64_t * signs64 = (const uint64_t *)keven_signs_q2xs; +// +// uint32_t aux32[4]; +// const uint8_t * aux8 = (const uint8_t *)aux32; +// +// float sumf = 0; +// +// for (int i = 0; i < nb; ++i) { +// const float d = GGML_FP16_TO_FP32(x[i].d) * y[i].d; +// const uint16_t * restrict q2 = x[i].qs; +// const int8_t * restrict q8 = y[i].qs; +// +// float sumf1 = 0, sumf2 = 0; +// +// for (int ib32 = 0; ib32 < QK_K/32; ib += 2) { +// int8x16_t q8b0 = vec_xl( 0, q8); +// int8x16_t qb81 = vec_xl(16, q8); +// int8x16_t q8b2 = vec_xl(32, q8); +// int8x16_t q8b3 = vec_xl(48, q8); +// q8 += 64; +// +// memcpy(aux32, q2, 4 * sizeof(uint32_t)); +// q2 += 8; +// +// int8x16_t q2u0 = { *(const int64_t *)(iq2xxs_grid + aux8[ 0]), *(const int64_t *)(iq2xxs_grid + aux8[ 1]) }; +// int8x16_t q2u1 = { *(const int64_t *)(iq2xxs_grid + aux8[ 2]), *(const int64_t *)(iq2xxs_grid + aux8[ 3]) }; +// int8x16_t q2u2 = { *(const int64_t *)(iq2xxs_grid + aux8[ 8]), *(const int64_t *)(iq2xxs_grid + aux8[ 9]) }; +// int8x16_t q2u3 = { *(const int64_t *)(iq2xxs_grid + aux8[10]), *(const int64_t *)(iq2xxs_grid + aux8[11]) }; +// +// int8x16_t q2s0 = { *(const int64_t *)(signs64 + ((aux32[1] >> 0) & 127)), *(const int64_t *)(signs64 + ((aux32[1] >> 7) & 127)) }; +// int8x16_t q2s1 = { *(const int64_t *)(signs64 + ((aux32[1] >> 14) & 127)), *(const int64_t *)(signs64 + ((aux32[1] >> 21) & 127)) }; +// int8x16_t q2s2 = { *(const int64_t *)(signs64 + ((aux32[3] >> 0) & 127)), *(const int64_t *)(signs64 + ((aux32[3] >> 7) & 127)) }; +// int8x16_t q2s3 = { *(const int64_t *)(signs64 + ((aux32[3] >> 14) & 127)), *(const int64_t *)(signs64 + ((aux32[3] >> 21) & 127)) }; +// +// q2u0 = vec_mul(q2u0, q2s0); +// q2u1 = vec_mul(q2u1, q2s1); +// q2u2 = vec_mul(q2u2, q2s2); +// q2u3 = vec_mul(q2u3, q2s3); +// +// const int32x4_t p1 = ggml_vec_dot(ggml_vec_dot(vec_splat_s32(0), q2u0, q8b0), q2u1, q8b1); +// const int32x4_t p2 = ggml_vec_dot(ggml_vec_dot(vec_splat_s32(0), q2u2, q8b2), q2u3, q8b3); +// +// sumf1 += (p1[0] + p1[1] + p1[2] + p1[3]) * (0.5f + (aux32[1] >> 28)); +// sumf2 += (p2[0] + p2[1] + p2[2] + p2[3]) * (0.5f + (aux32[3] >> 28)); +// } +// +// sumf += d * (sumf1 + sumf2); +// } +// +// *s = 0.25f * sumf; #else uint32_t aux32[2]; From eb3fa5d9c40c966d48c0902c7bd98799aa94780b Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Mon, 20 Jan 2025 11:19:34 +0800 Subject: [PATCH 30/56] ggml: update ggml_vec_dot_q4_1_q8_1 to use typedefs Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 33 ++++++++++++++++------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 039963161fea4..b202953dd3b16 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -2694,29 +2694,32 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r sumf = hsum_float_8(acc) + summs; #elif defined(__VXE__) || defined(__VXE2__) float summs = 0; - __vector float acc = vec_splats(0.0f); + float32x4_t acc = vec_splats(0.0f); - const __vector uint8_t v_m = vec_splats((const uint8_t)0x0F); + const uint8x16_t v_m = vec_splat_u8(0x0F); for (; ib < nb; ++ib) { - summs += GGML_FP16_TO_FP32(x[ib].m) * GGML_FP16_TO_FP32(y[ib].s); + const block_q4_1 * restrict x0 = &x[ib]; + const block_q8_1 * restrict y0 = &y[ib]; - const __vector uint8_t v_x = vec_xl(0, x[ib].qs); - const __vector int8_t v_xl = (const __vector int8_t)(v_x & v_m); - const __vector int8_t v_xh = (const __vector int8_t)(v_x >> 4); + summs += GGML_FP16_TO_FP32(x0->m) * GGML_FP16_TO_FP32(y0->s); - const __vector int8_t v_yl = vec_xl(0 , y[ib].qs); - const __vector int8_t v_yh = vec_xl(QK8_1/2, y[ib].qs); + const uint8x16_t v_x = vec_xl(0, x0->qs); + const int8x16_t v_xl = (const int8x16_t)(v_x & v_m); + const int8x16_t v_xh = (const int8x16_t)(v_x >> 4); - const __vector int16_t xylo = vec_mulo(v_xl, v_yl); - const __vector int16_t xyle = vec_mule(v_xl, v_yl); - const __vector int16_t xyho = vec_mulo(v_xh, v_yh); - const __vector int16_t xyhe = vec_mule(v_xh, v_yh); + const int8x16_t v_yl = vec_xl(0 , y0->qs); + const int8x16_t v_yh = vec_xl(QK8_1/2, y0->qs); - __vector int16_t v_xy_ = xylo + xyle + xyho + xyhe; v_xy_ += vec_reve(v_xy_); + const int16x8_t xylo = vec_mulo(v_xl, v_yl); + const int16x8_t xyle = vec_mule(v_xl, v_yl); + const int16x8_t xyho = vec_mulo(v_xh, v_yh); + const int16x8_t xyhe = vec_mule(v_xh, v_yh); - const __vector float v_xy = vec_float(vec_unpackh(v_xy_)); - const __vector float v_d = vec_splats(GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d)); + int16x8_t v_xy_ = xylo + xyle + xyho + xyhe; v_xy_ += vec_reve(v_xy_); + + const float32x4_t v_xy = vec_float(vec_unpackh(v_xy_)); + const float32x4_t v_d = vec_splats(GGML_FP16_TO_FP32(x0->d) * GGML_FP16_TO_FP32(y0->d)); acc = vec_madd(v_xy, v_d, acc); } From 33f98bd78f42c5811a9b9bedd02943b4c3706ca0 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Mon, 20 Jan 2025 13:32:53 +0800 Subject: [PATCH 31/56] ggml: switch to restrict for iq4_nl Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index b202953dd3b16..a474b1963a4e3 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -10688,18 +10688,21 @@ void ggml_vec_dot_iq4_nl_q8_0(int n, float * restrict s, size_t bs, const void * const uint8x16_t v_m = vec_splat_u8(0x0F); for (; ib < nb; ++ib) { - const uint8x16_t v_x = vec_xl(0, x[ib].qs); + const block_iq4_nl * restrict x0 = &x[ib]; + const block_q8_0 * restrict y0 = &y[ib]; + + const uint8x16_t v_x = vec_xl(0, x0->qs); int8x16_t v_xl = (int8x16_t)vec_and(v_x, v_m); int8x16_t v_xh = (int8x16_t)vec_sr(v_x, 4); v_xl = vec_perm(v_k, v_k, (uchar1x16_t)v_xl); v_xh = vec_perm(v_k, v_k, (uchar1x16_t)v_xh); - const int8x16_t v_yl = vec_xl( 0, y[ib].qs); - const int8x16_t v_yh = vec_xl(QK8_0/2, y[ib].qs); + const int8x16_t v_yl = vec_xl(0 , y0->qs); + const int8x16_t v_yh = vec_xl(QK8_0/2, y0->qs); const int32x4_t v_xy = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh); - sumf += GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d) * (v_xy[0] + v_xy[1] + v_xy[2] + v_xy[3]); + sumf += GGML_FP16_TO_FP32(x0->d) * GGML_FP16_TO_FP32(y0->d) * (v_xy[0] + v_xy[1] + v_xy[2] + v_xy[3]); } #endif for (; ib < nb; ++ib) { From 948441c22704be1a43ce7ef02c26c862cb8d37d8 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Mon, 20 Jan 2025 13:45:21 +0800 Subject: [PATCH 32/56] ggml: slight dot product speed improvement for q4_1_q8_1 Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index a474b1963a4e3..530d51dbb4049 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -2711,14 +2711,9 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r const int8x16_t v_yl = vec_xl(0 , y0->qs); const int8x16_t v_yh = vec_xl(QK8_1/2, y0->qs); - const int16x8_t xylo = vec_mulo(v_xl, v_yl); - const int16x8_t xyle = vec_mule(v_xl, v_yl); - const int16x8_t xyho = vec_mulo(v_xh, v_yh); - const int16x8_t xyhe = vec_mule(v_xh, v_yh); + const int32x4_t v_xy_ = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh); + const float32x4_t v_xy = vec_float(v_xy_); - int16x8_t v_xy_ = xylo + xyle + xyho + xyhe; v_xy_ += vec_reve(v_xy_); - - const float32x4_t v_xy = vec_float(vec_unpackh(v_xy_)); const float32x4_t v_d = vec_splats(GGML_FP16_TO_FP32(x0->d) * GGML_FP16_TO_FP32(y0->d)); acc = vec_madd(v_xy, v_d, acc); From 9a391471f6e22ef6f47fc72cc2ff0dd73c13c079 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Mon, 20 Jan 2025 21:43:56 +0800 Subject: [PATCH 33/56] ggml: s390x SIMD activation for q6_K Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-impl.h | 46 +++++++++++--- ggml/src/ggml-cpu/ggml-cpu-quants.c | 94 +++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+), 9 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-impl.h b/ggml/src/ggml-cpu/ggml-cpu-impl.h index 617fd7133147e..e8556f7d2c834 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-impl.h +++ b/ggml/src/ggml-cpu/ggml-cpu-impl.h @@ -411,25 +411,40 @@ typedef double double64x2_t __attribute((vector_size(16))); typedef signed long long long64x2_t __attribute((vector_size(16))); typedef unsigned long long ulong64x2_t __attribute__((vector_size(16))); -typedef struct ggml_uint8x2_t { +typedef struct ggml_uint8x16x2_t { uint8x16_t val[2]; -} ggml_uint8x2_t; +} ggml_uint8x16x2_t; -typedef struct ggml_int8x4_t { - int8x16_t val[4]; -} ggml_int8x4_t; +inline static ggml_uint8x16x2_t ggml_vec_xl_u8x2(const uint8_t * ptr) { + ggml_uint8x16x2_t res; + + res.val[0] = vec_xl( 0, ptr); + res.val[1] = vec_xl(16, ptr); + + return res; +} + +typedef struct ggml_uint8x16x4_t { + uint8x16_t val[4]; +} ggml_uint8x16x4_t; -inline static ggml_uint8x2_t ggml_vec_xl_x2(const uint8_t * ptr) { - ggml_uint8x2_t res; +inline static ggml_uint8x16x4_t ggml_vec_xl_u8x4(const uint8_t * ptr) { + ggml_uint8x16x4_t res; res.val[0] = vec_xl( 0, ptr); res.val[1] = vec_xl(16, ptr); + res.val[2] = vec_xl(32, ptr); + res.val[3] = vec_xl(48, ptr); return res; } -inline static ggml_int8x4_t ggml_vec_xl_x4(const int8_t * ptr) { - ggml_int8x4_t res; +typedef struct ggml_int8x16x4 { + int8x16_t val[4]; +} ggml_int8x16x4; + +inline static ggml_int8x16x4 ggml_vec_xl_s8x4(const int8_t * ptr) { + ggml_int8x16x4 res; res.val[0] = vec_xl( 0, ptr); res.val[1] = vec_xl(16, ptr); @@ -439,6 +454,19 @@ inline static ggml_int8x4_t ggml_vec_xl_x4(const int8_t * ptr) { return res; } +typedef struct ggml_int16x8x2_t { + int16x8_t val[2]; +} ggml_int16x8x2_t; + +inline static ggml_int16x8x2_t ggml_vec_xl_s16x2(const int16_t * ptr) { + ggml_int16x8x2_t res; + + res.val[0] = vec_xl( 0, ptr); + res.val[1] = vec_xl(16, ptr); + + return res; +} + /* ! WARNING: Very slow. Use vec_perm if possible. Refer to iq4_xs ! or iq4_nl for example implementation. diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 530d51dbb4049..638af7b86e677 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -7502,7 +7502,101 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * r } *s = hsum_float_8(acc); +#elif defined(__VXE__) || defined(__VXE2__) + float sum = 0; + + const uint8x16_t m4b = vec_splat_u8(0x0F); + const int32x4_t vzero = vec_splat_s32(0); + const uint8x16_t mone = vec_splat_u8(3); + + ggml_int8x16x4_t q6bytes; + ggml_uint8x16x4_t q6h; + + for (int i = 0; i < nb; ++i) { + const float d_all = GGML_FP16_TO_FP32(x[i].d); + const uint8_t * restrict q6 = x[i].ql; + const uint8_t * restrict qh = x[i].qh; + const int8_t * restrict q8 = y[i].qs; + + const int8_t * restrict scale = x[i].scales; + + const ggml_int16x8x2_t q8sums = ggml_vec_xl_s16x2(y[i].bsums); + const int8x16_t scales = vec_xl(0, scale); + const ggml_int16x8x2_t q6scales = {{ vec_unpackh(scales), vec_unpackl(scales) }}; + + const int32x4_t q8q6lo = vec_mulo(q8sums.val[0], q6scales.val[0]); + const int32x4_t q8q6le = vec_mule(q8sums.val[0], q6scales.val[0]); + const int32x4_t q8q6ho = vec_mulo(q8sums.val[1], q6scales.val[1]); + const int32x4_t q8q6he = vec_mule(q8sums.val[1], q6scales.val[1]); + const int32x4_t q8q6 = q8q6lo + q8q6le + q8q6ho + q8q6he; + + const int32_t isum_mins = q8q6[0] + q8q6[1] + q8q6[2] + q8q6[3]; + + int32_t isum = 0; + + for (int j = 0; j < QK_K/128; ++j) { + ggml_uint8x16x2_t qhbits = ggml_vec_xl_u8x2(qh); qh += 32; + ggml_uint8x16x4_t q6bits = ggml_vec_xl_u8x4(q6); q6 += 64; + ggml_int8x16x4_t q8bytes = ggml_vec_xl_s8x4(q8); q8 += 64; + + q6h.val[0] = vec_sl(vec_and(mone, qhbits.val[0]), 4); + q6h.val[1] = vec_sl(vec_and(mone, qhbits.val[1]), 4); + uint8x16_t shifted = vec_sr(qhbits.val[0], 2); + q6h.val[2] = vec_sl(vec_and(mone, shifted), 4); + shifted = vec_sr(qhbits.val[1], 2); + q6h.val[3] = vec_sl(vec_and(mone, shifted), 4); + + q6bytes.val[0] = (int8x16_t)(vec_or(vec_and(q6bits.val[0], m4b), q6h.val[0])); + q6bytes.val[1] = (int8x16_t)(vec_or(vec_and(q6bits.val[1], m4b), q6h.val[1])); + q6bytes.val[2] = (int8x16_t)(vec_or(vec_and(q6bits.val[2], m4b), q6h.val[2])); + q6bytes.val[3] = (int8x16_t)(vec_or(vec_and(q6bits.val[3], m4b), q6h.val[3])); + + int32x4_t summs0 = ggml_vec_dot(vzero, q6bytes.val[0], q8bytes.val[0]); + int32x4_t summs1 = ggml_vec_dot(vzero, q6bytes.val[1], q8bytes.val[1]); + int32x4_t summs2 = ggml_vec_dot(vzero, q6bytes.val[2], q8bytes.val[2]); + int32x4_t summs3 = ggml_vec_dot(vzero, q6bytes.val[3], q8bytes.val[3]); + + isum += (summs0[0] + summs0[1] + summs0[2] + summs0[3]) * scale[0] + + (summs1[0] + summs1[1] + summs1[2] + summs1[3]) * scale[1] + + (summs2[0] + summs2[1] + summs2[2] + summs2[3]) * scale[2] + + (summs3[0] + summs3[1] + summs3[2] + summs3[3]) * scale[3]; + + scale += 4; + + q8bytes = ggml_vec_xl_s8x4(q8); q8 += 64; + + shifted = vec_sr(qhbits.val[0], 4); + q6h.val[0] = vec_sl(vec_and(mone, shifted), 4); + shifted = vec_sr(qhbits.val[1], 4); + q6h.val[1] = vec_sl(vec_and(mone, shifted), 4); + shifted = vec_sr(qhbits.val[0], 6); + q6h.val[2] = vec_sl(vec_and(mone, shifted), 4); + shifted = vec_sr(qhbits.val[1], 6); + q6h.val[3] = vec_sl(vec_and(mone, shifted), 4); + + q6bytes.val[0] = (int8x16_t)(vec_or(vec_sr(q6bits.val[0], 4), q6h.val[0])); + q6bytes.val[1] = (int8x16_t)(vec_or(vec_sr(q6bits.val[1], 4), q6h.val[1])); + q6bytes.val[2] = (int8x16_t)(vec_or(vec_sr(q6bits.val[2], 4), q6h.val[2])); + q6bytes.val[3] = (int8x16_t)(vec_or(vec_sr(q6bits.val[3], 4), q6h.val[3])); + + summs0 = ggml_vec_dot(vzero, q6bytes.val[0], q8bytes.val[0]); + summs1 = ggml_vec_dot(vzero, q6bytes.val[1], q8bytes.val[1]); + summs2 = ggml_vec_dot(vzero, q6bytes.val[2], q8bytes.val[2]); + summs3 = ggml_vec_dot(vzero, q6bytes.val[3], q8bytes.val[3]); + + isum += (summs0[0] + summs0[1] + summs0[2] + summs0[3]) * scale[0] + + (summs1[0] + summs1[1] + summs1[2] + summs1[3]) * scale[1] + + (summs2[0] + summs2[1] + summs2[2] + summs2[3]) * scale[2] + + (summs3[0] + summs3[1] + summs3[2] + summs3[3]) * scale[3]; + + scale += 4; + } + + sum += d_all * y[i].d * (isum - 32 * isum_mins); + } + + *s = sum; #else int8_t aux8[QK_K]; From 87087de69eab948783b794cffcd2e4bc4a03f059 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Mon, 20 Jan 2025 21:49:24 +0800 Subject: [PATCH 34/56] ggml: add missing `_t` to ggml_int8x16x4_t Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-impl.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-impl.h b/ggml/src/ggml-cpu/ggml-cpu-impl.h index e8556f7d2c834..8604872b94055 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-impl.h +++ b/ggml/src/ggml-cpu/ggml-cpu-impl.h @@ -439,11 +439,11 @@ inline static ggml_uint8x16x4_t ggml_vec_xl_u8x4(const uint8_t * ptr) { return res; } -typedef struct ggml_int8x16x4 { +typedef struct ggml_int8x16x4_t { int8x16_t val[4]; } ggml_int8x16x4; -inline static ggml_int8x16x4 ggml_vec_xl_s8x4(const int8_t * ptr) { +inline static ggml_int8x16x4_t ggml_vec_xl_s8x4(const int8_t * ptr) { ggml_int8x16x4 res; res.val[0] = vec_xl( 0, ptr); From 077a597951e0ec6608dbab95d644bc476ad698f9 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Mon, 20 Jan 2025 21:52:46 +0800 Subject: [PATCH 35/56] ggml: fix missing `_t` for ggml_vec_xl_s8x4 Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-impl.h b/ggml/src/ggml-cpu/ggml-cpu-impl.h index 8604872b94055..2401a5e419e43 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-impl.h +++ b/ggml/src/ggml-cpu/ggml-cpu-impl.h @@ -444,7 +444,7 @@ typedef struct ggml_int8x16x4_t { } ggml_int8x16x4; inline static ggml_int8x16x4_t ggml_vec_xl_s8x4(const int8_t * ptr) { - ggml_int8x16x4 res; + ggml_int8x16x4_t res; res.val[0] = vec_xl( 0, ptr); res.val[1] = vec_xl(16, ptr); From 9210d7061a32104a92810e7904fdc8603b7df8e5 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Mon, 20 Jan 2025 21:57:56 +0800 Subject: [PATCH 36/56] ggml: fix more missing `_t` Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-impl.h b/ggml/src/ggml-cpu/ggml-cpu-impl.h index 2401a5e419e43..3cc86e3902a04 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-impl.h +++ b/ggml/src/ggml-cpu/ggml-cpu-impl.h @@ -441,7 +441,7 @@ inline static ggml_uint8x16x4_t ggml_vec_xl_u8x4(const uint8_t * ptr) { typedef struct ggml_int8x16x4_t { int8x16_t val[4]; -} ggml_int8x16x4; +} ggml_int8x16x4_t; inline static ggml_int8x16x4_t ggml_vec_xl_s8x4(const int8_t * ptr) { ggml_int8x16x4_t res; From 59d2638435d1bbcf100230dd7e30526549a3c456 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Wed, 5 Feb 2025 19:03:16 +0800 Subject: [PATCH 37/56] ggml: add unroll and prefetch to Q8_0 increase of 3.86% for prompt processing and 32.22% for token generation Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 638af7b86e677..14ed6f80ee06e 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -3818,7 +3818,11 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * r #elif defined(__VXE__) || defined(__VXE2__) __vector float acc = vec_splats(0.0f); +#pragma GCC unroll 8 for (; ib < nb; ++ib) { + __builtin_prefetch(x[ib].qs, 0, 1); + __builtin_prefetch(y[ib].qs, 0, 1); + const __vector int8_t v_xl = vec_xl(0 , x[ib].qs); const __vector int8_t v_xh = vec_xl(QK8_0/2, x[ib].qs); const __vector int8_t v_yl = vec_xl(0 , y[ib].qs); From 5c5e0aa58c99bdef74425467f11e0b5d5524b834 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Wed, 5 Feb 2025 21:01:47 +0800 Subject: [PATCH 38/56] ggml: patch Q8_0 to use proper vector sizes Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 14ed6f80ee06e..ad69a166c14b8 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -3823,20 +3823,20 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * r __builtin_prefetch(x[ib].qs, 0, 1); __builtin_prefetch(y[ib].qs, 0, 1); - const __vector int8_t v_xl = vec_xl(0 , x[ib].qs); - const __vector int8_t v_xh = vec_xl(QK8_0/2, x[ib].qs); - const __vector int8_t v_yl = vec_xl(0 , y[ib].qs); - const __vector int8_t v_yh = vec_xl(QK8_0/2, y[ib].qs); + const int8x16_t v_xl = vec_xl(0 , x[ib].qs); + const int8x16_t v_xh = vec_xl(QK8_0/2, x[ib].qs); + const int8x16_t v_yl = vec_xl(0 , y[ib].qs); + const int8x16_t v_yh = vec_xl(QK8_0/2, y[ib].qs); - const __vector int16_t v_xylo = vec_mulo(v_xl, v_yl); - const __vector int16_t v_xyle = vec_mule(v_xl, v_yl); - const __vector int16_t v_xyho = vec_mulo(v_xh, v_yh); - const __vector int16_t v_xyhe = vec_mule(v_xh, v_yh); + const int16x8_t v_xylo = vec_mulo(v_xl, v_yl); + const int16x8_t v_xyle = vec_mule(v_xl, v_yl); + const int16x8_t v_xyho = vec_mulo(v_xh, v_yh); + const int16x8_t v_xyhe = vec_mule(v_xh, v_yh); - __vector int16_t v_xy_ = v_xylo + v_xyle + v_xyho + v_xyhe; v_xy_ += vec_reve(v_xy_); + int16x8_t v_xy_ = v_xylo + v_xyle + v_xyho + v_xyhe; v_xy_ += vec_reve(v_xy_); - const __vector float v_xy = vec_float(vec_unpackh(v_xy_)); - const __vector float v_d = vec_splats(GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d)); + const float32x4_t v_xy = vec_float(vec_unpackh(v_xy_)); + const float32x4_t v_d = vec_splats(GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d)); acc = vec_madd(v_xy, v_d, acc); } From 69d86951b7aa076bab45d985ced9e83c952d6d7e Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Wed, 5 Feb 2025 21:12:34 +0800 Subject: [PATCH 39/56] ggml: optimise Q8_0 dot prod compute kernel further Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index ad69a166c14b8..c65bd52aa5e3a 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -3828,14 +3828,8 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * restrict s, size_t bs, const void * r const int8x16_t v_yl = vec_xl(0 , y[ib].qs); const int8x16_t v_yh = vec_xl(QK8_0/2, y[ib].qs); - const int16x8_t v_xylo = vec_mulo(v_xl, v_yl); - const int16x8_t v_xyle = vec_mule(v_xl, v_yl); - const int16x8_t v_xyho = vec_mulo(v_xh, v_yh); - const int16x8_t v_xyhe = vec_mule(v_xh, v_yh); - - int16x8_t v_xy_ = v_xylo + v_xyle + v_xyho + v_xyhe; v_xy_ += vec_reve(v_xy_); - - const float32x4_t v_xy = vec_float(vec_unpackh(v_xy_)); + const int32x4_t v_xy_ = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh); + const float32x4_t v_xy = vec_float(v_xy_); const float32x4_t v_d = vec_splats(GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d)); acc = vec_madd(v_xy, v_d, acc); From b11ffbdf6d72a192b5653d36b9d3eab088d99be7 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Thu, 6 Feb 2025 17:59:27 +0800 Subject: [PATCH 40/56] ggml: add unroll and prefetch to Q4_1 Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index c65bd52aa5e3a..e020d5a200514 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -2698,23 +2698,24 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * restrict s, size_t bs, const void * r const uint8x16_t v_m = vec_splat_u8(0x0F); +#pragma GCC unroll 4 for (; ib < nb; ++ib) { - const block_q4_1 * restrict x0 = &x[ib]; - const block_q8_1 * restrict y0 = &y[ib]; + __builtin_prefetch(x[ib].qs, 0, 1); + __builtin_prefetch(y[ib].qs, 0, 1); - summs += GGML_FP16_TO_FP32(x0->m) * GGML_FP16_TO_FP32(y0->s); + summs += GGML_FP16_TO_FP32(x[ib].m) * GGML_FP16_TO_FP32(y[ib].s); - const uint8x16_t v_x = vec_xl(0, x0->qs); + const uint8x16_t v_x = vec_xl(0, x[ib].qs); const int8x16_t v_xl = (const int8x16_t)(v_x & v_m); const int8x16_t v_xh = (const int8x16_t)(v_x >> 4); - const int8x16_t v_yl = vec_xl(0 , y0->qs); - const int8x16_t v_yh = vec_xl(QK8_1/2, y0->qs); + const int8x16_t v_yl = vec_xl(0 , y[ib].qs); + const int8x16_t v_yh = vec_xl(QK8_1/2, y[ib].qs); const int32x4_t v_xy_ = ggml_vec_dot(ggml_vec_dot(vec_splats(0), v_xl, v_yl), v_xh, v_yh); const float32x4_t v_xy = vec_float(v_xy_); - const float32x4_t v_d = vec_splats(GGML_FP16_TO_FP32(x0->d) * GGML_FP16_TO_FP32(y0->d)); + const float32x4_t v_d = vec_splats(GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d)); acc = vec_madd(v_xy, v_d, acc); } From dac5d9e512b0d6e95d2a8618f25c6c4c1cd558aa Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Fri, 7 Feb 2025 18:41:13 +0800 Subject: [PATCH 41/56] ggml: refactor Q6_K variable naming for readability Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 145 +++++++++++++++++----------- 1 file changed, 87 insertions(+), 58 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index e020d5a200514..8781333e87625 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -7504,57 +7504,80 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * r #elif defined(__VXE__) || defined(__VXE2__) float sum = 0; - const uint8x16_t m4b = vec_splat_u8(0x0F); - const int32x4_t vzero = vec_splat_s32(0); - const uint8x16_t mone = vec_splat_u8(3); + // Lower 4-bit and upper 2-bit masks + const uint8x16_t v_lm = vec_splat_u8(0x0F); + const uint8x16_t v_um = vec_splat_u8(0x03); - ggml_int8x16x4_t q6bytes; - ggml_uint8x16x4_t q6h; + const int32x4_t v_z = vec_splat_s32(0); + + int8x16_t q6b[4]; + uint8x16_t q6h[4]; + + uint8x16_t v_xl[4]; + uint8x16_t v_xh[2]; + int8x16_t v_y[4]; for (int i = 0; i < nb; ++i) { const float d_all = GGML_FP16_TO_FP32(x[i].d); - const uint8_t * restrict q6 = x[i].ql; - const uint8_t * restrict qh = x[i].qh; - const int8_t * restrict q8 = y[i].qs; + const uint8_t * restrict x0l = x[i].ql; + const uint8_t * restrict x0h = x[i].qh; + const int8_t * restrict y0 = y[i].qs; const int8_t * restrict scale = x[i].scales; - const ggml_int16x8x2_t q8sums = ggml_vec_xl_s16x2(y[i].bsums); - const int8x16_t scales = vec_xl(0, scale); - const ggml_int16x8x2_t q6scales = {{ vec_unpackh(scales), vec_unpackl(scales) }}; + const int16x8_t v_ysumsl = vec_xl(0 , y[i].bsums); + const int16x8_t v_ysumsh = vec_xl(16, y[i].bsums); - const int32x4_t q8q6lo = vec_mulo(q8sums.val[0], q6scales.val[0]); - const int32x4_t q8q6le = vec_mule(q8sums.val[0], q6scales.val[0]); - const int32x4_t q8q6ho = vec_mulo(q8sums.val[1], q6scales.val[1]); - const int32x4_t q8q6he = vec_mule(q8sums.val[1], q6scales.val[1]); - const int32x4_t q8q6 = q8q6lo + q8q6le + q8q6ho + q8q6he; + const int8x16_t v_scale = vec_xl(0, scale); + const int16x8_t v_scalel = vec_unpackh(scales); + const int16x8_t v_scaleh = vec_unpackl(scales); - const int32_t isum_mins = q8q6[0] + q8q6[1] + q8q6[2] + q8q6[3]; + const int32x4_t v_minslo = vec_mulo(v_ysumsl, v_scalel); + const int32x4_t v_minsle = vec_mule(v_ysumsl, v_scalel); + const int32x4_t v_minsho = vec_mulo(v_ysumsh, v_scaleh); + const int32x4_t v_minshe = vec_mule(v_ysumsh, v_scaleh); + const int32x4_t v_mins = v_minslo + v_minsle + v_minsho + v_minshe; - int32_t isum = 0; + const int32_t mins = vmins[0] + vmins[1] + vmins[2] + vmins[3]; + int32_t isum = 0; for (int j = 0; j < QK_K/128; ++j) { - ggml_uint8x16x2_t qhbits = ggml_vec_xl_u8x2(qh); qh += 32; - ggml_uint8x16x4_t q6bits = ggml_vec_xl_u8x4(q6); q6 += 64; - ggml_int8x16x4_t q8bytes = ggml_vec_xl_s8x4(q8); q8 += 64; - - q6h.val[0] = vec_sl(vec_and(mone, qhbits.val[0]), 4); - q6h.val[1] = vec_sl(vec_and(mone, qhbits.val[1]), 4); - uint8x16_t shifted = vec_sr(qhbits.val[0], 2); - q6h.val[2] = vec_sl(vec_and(mone, shifted), 4); - shifted = vec_sr(qhbits.val[1], 2); - q6h.val[3] = vec_sl(vec_and(mone, shifted), 4); - - q6bytes.val[0] = (int8x16_t)(vec_or(vec_and(q6bits.val[0], m4b), q6h.val[0])); - q6bytes.val[1] = (int8x16_t)(vec_or(vec_and(q6bits.val[1], m4b), q6h.val[1])); - q6bytes.val[2] = (int8x16_t)(vec_or(vec_and(q6bits.val[2], m4b), q6h.val[2])); - q6bytes.val[3] = (int8x16_t)(vec_or(vec_and(q6bits.val[3], m4b), q6h.val[3])); - - int32x4_t summs0 = ggml_vec_dot(vzero, q6bytes.val[0], q8bytes.val[0]); - int32x4_t summs1 = ggml_vec_dot(vzero, q6bytes.val[1], q8bytes.val[1]); - int32x4_t summs2 = ggml_vec_dot(vzero, q6bytes.val[2], q8bytes.val[2]); - int32x4_t summs3 = ggml_vec_dot(vzero, q6bytes.val[3], q8bytes.val[3]); + // Load model upper 2 bits + v_xh[0] = vec_xl(0 , x0h); + v_xh[1] = vec_xl(16, x0h); + x0h += 32; + + // Load model lower 4 bits + v_xl[0] = vec_xl(0 , x0l); + v_xl[1] = vec_xl(16, x0l); + v_xl[2] = vec_xl(32, x0l); + v_xl[3] = vec_xl(48, x0l); + x0l += 64; + + // Load activation quants + v_y[0] = vec_xl(0 , y0); + v_y[1] = vec_xl(16, y0); + v_y[2] = vec_xl(32, y0); + v_y[3] = vec_xl(48, y0); + y0 += 64; + + q6h[0] = vec_sl(vec_and(v_um, v_xh[0]), 4); + q6h[1] = vec_sl(vec_and(v_um, v_xh[1]), 4); + uint8x16_t shifted = vec_sr(v_xh[0], 2); + q6h[2] = vec_sl(vec_and(v_um, shifted), 4); + shifted = vec_sr(v_xh[1], 2); + q6h[3] = vec_sl(vec_and(v_um, shifted), 4); + + q6b[0] = (int8x16_t)(vec_or(vec_and(v_xl[0], v_lm), q6h[0])); + q6b[1] = (int8x16_t)(vec_or(vec_and(v_xl[1], v_lm), q6h[1])); + q6b[2] = (int8x16_t)(vec_or(vec_and(v_xl[2], v_lm), q6h[2])); + q6b[3] = (int8x16_t)(vec_or(vec_and(v_xl[3], v_lm), q6h[3])); + + int32x4_t summs0 = ggml_vec_dot(v_z, q6b[0], v_y[0]); + int32x4_t summs1 = ggml_vec_dot(v_z, q6b[1], v_y[1]); + int32x4_t summs2 = ggml_vec_dot(v_z, q6b[2], v_y[2]); + int32x4_t summs3 = ggml_vec_dot(v_z, q6b[3], v_y[3]); isum += (summs0[0] + summs0[1] + summs0[2] + summs0[3]) * scale[0] + (summs1[0] + summs1[1] + summs1[2] + summs1[3]) * scale[1] + @@ -7563,26 +7586,32 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * r scale += 4; - q8bytes = ggml_vec_xl_s8x4(q8); q8 += 64; - shifted = vec_sr(qhbits.val[0], 4); - q6h.val[0] = vec_sl(vec_and(mone, shifted), 4); - shifted = vec_sr(qhbits.val[1], 4); - q6h.val[1] = vec_sl(vec_and(mone, shifted), 4); - shifted = vec_sr(qhbits.val[0], 6); - q6h.val[2] = vec_sl(vec_and(mone, shifted), 4); - shifted = vec_sr(qhbits.val[1], 6); - q6h.val[3] = vec_sl(vec_and(mone, shifted), 4); - - q6bytes.val[0] = (int8x16_t)(vec_or(vec_sr(q6bits.val[0], 4), q6h.val[0])); - q6bytes.val[1] = (int8x16_t)(vec_or(vec_sr(q6bits.val[1], 4), q6h.val[1])); - q6bytes.val[2] = (int8x16_t)(vec_or(vec_sr(q6bits.val[2], 4), q6h.val[2])); - q6bytes.val[3] = (int8x16_t)(vec_or(vec_sr(q6bits.val[3], 4), q6h.val[3])); - - summs0 = ggml_vec_dot(vzero, q6bytes.val[0], q8bytes.val[0]); - summs1 = ggml_vec_dot(vzero, q6bytes.val[1], q8bytes.val[1]); - summs2 = ggml_vec_dot(vzero, q6bytes.val[2], q8bytes.val[2]); - summs3 = ggml_vec_dot(vzero, q6bytes.val[3], q8bytes.val[3]); + // Load activation quants + v_y[0] = vec_xl(0 , y0); + v_y[1] = vec_xl(16, y0); + v_y[2] = vec_xl(32, y0); + v_y[3] = vec_xl(48, y0); + y0 += 64; + + shifted = vec_sr(v_xh[0], 4); + q6h[0] = vec_sl(vec_and(v_um, shifted), 4); + shifted = vec_sr(v_xh[1], 4); + q6h[1] = vec_sl(vec_and(v_um, shifted), 4); + shifted = vec_sr(v_xh[0], 6); + q6h[2] = vec_sl(vec_and(v_um, shifted), 4); + shifted = vec_sr(v_xh[1], 6); + q6h[3] = vec_sl(vec_and(v_um, shifted), 4); + + q6b[0] = (int8x16_t)(vec_or(vec_sr(v_xl[0], 4), q6h[0])); + q6b[1] = (int8x16_t)(vec_or(vec_sr(v_xl[1], 4), q6h[1])); + q6b[2] = (int8x16_t)(vec_or(vec_sr(v_xl[2], 4), q6h[2])); + q6b[3] = (int8x16_t)(vec_or(vec_sr(v_xl[3], 4), q6h[3])); + + summs0 = ggml_vec_dot(v_z, q6b[0], v_y[0]); + summs1 = ggml_vec_dot(v_z, q6b[1], v_y[1]); + summs2 = ggml_vec_dot(v_z, q6b[2], v_y[2]); + summs3 = ggml_vec_dot(v_z, q6b[3], v_y[3]); isum += (summs0[0] + summs0[1] + summs0[2] + summs0[3]) * scale[0] + (summs1[0] + summs1[1] + summs1[2] + summs1[3]) * scale[1] + @@ -7592,7 +7621,7 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * r scale += 4; } - sum += d_all * y[i].d * (isum - 32 * isum_mins); + sum += d_all * y[i].d * (isum - 32 * mins); } *s = sum; From 8fe0803dafb40a8c1481279262efc4596b73fce4 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Fri, 7 Feb 2025 18:44:12 +0800 Subject: [PATCH 42/56] ggml: fix Q6_K typos Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 8781333e87625..18b90b112c9d6 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -7530,8 +7530,8 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * r const int16x8_t v_ysumsh = vec_xl(16, y[i].bsums); const int8x16_t v_scale = vec_xl(0, scale); - const int16x8_t v_scalel = vec_unpackh(scales); - const int16x8_t v_scaleh = vec_unpackl(scales); + const int16x8_t v_scalel = vec_unpackh(v_scale); + const int16x8_t v_scaleh = vec_unpackl(v_scale); const int32x4_t v_minslo = vec_mulo(v_ysumsl, v_scalel); const int32x4_t v_minsle = vec_mule(v_ysumsl, v_scalel); @@ -7539,7 +7539,7 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * r const int32x4_t v_minshe = vec_mule(v_ysumsh, v_scaleh); const int32x4_t v_mins = v_minslo + v_minsle + v_minsho + v_minshe; - const int32_t mins = vmins[0] + vmins[1] + vmins[2] + vmins[3]; + const int32_t mins = v_mins[0] + v_mins[1] + v_mins[2] + v_mins[3]; int32_t isum = 0; for (int j = 0; j < QK_K/128; ++j) { From 333e1a2697df67e64c41ffe249ac4abc58bd0a64 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Sun, 9 Feb 2025 20:29:44 +0800 Subject: [PATCH 43/56] ggml: s390x SIMD activation for Q5_K Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-impl.h | 24 ++++++++ ggml/src/ggml-cpu/ggml-cpu-quants.c | 87 +++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+) diff --git a/ggml/src/ggml-cpu/ggml-cpu-impl.h b/ggml/src/ggml-cpu/ggml-cpu-impl.h index 3cc86e3902a04..498a145f347f8 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-impl.h +++ b/ggml/src/ggml-cpu/ggml-cpu-impl.h @@ -494,6 +494,30 @@ inline static int8x16_t ggml_vec_tbl(int8x16_t a, uint8x16_t b) { return res; } +inline static int16x8_t vec_padd_s16(int16x8_t a, int16x8_t b) { + const uchar8x16_t v_maske = { + 0x00, 0x01, 0x04, 0x05, 0x08, 0x09, 0x0C, 0x0D, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF + }; + + const uchar8x16_t v_masko = { + 0x02, 0x03, 0x06, 0x07, 0x0A, 0x0B, 0x0E, 0x0F, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF + }; + + const uchar8x16_t v_maskj = { + 0, 1, 2, 3, 4, 5, 6, 7, + 16, 17, 18, 19, 20, 21, 22, 23 + }; + + const int16x8_t pa = vec_add(vec_perm(a, a, v_maske), + vec_perm(a, a, v_masko)); + const int16x8_t pb = vec_add(vec_perm(b, b, v_maske), + vec_perm(b, b, v_masko)); + + return vec_perm(pa, pb, v_maskj); +} + inline static int32x4_t ggml_vec_dot(int32x4_t acc, int8x16_t a, int8x16_t b) { const int16x8_t p = vec_mule(a, b) + vec_mulo(a, b); return acc + (vec_unpackh(p) + vec_unpackl(p)); diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 18b90b112c9d6..c274c79aacb80 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -6845,7 +6845,94 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r } *s = hsum_float_8(acc) + summs; +#elif defined(__VXE__) || defined(__VXE2__) + const uint8x16_t v_lm = vec_splat_u8(0x0F); + const uint8x16_t v_1m = vec_splat_u8(0x01); + const uint8x16_t v_2m = vec_splat_u8(0x02); + + const int32x4_t v_z = vec_splat_s32(0); + + const uchar8x16_t v_minsm = { + 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF + }; + + int8x16_t q5b[4]; + uint8x16_t q5h[4]; + + uint8x16_t v_xl[2]; + uint8x16_t v_xh[2]; + int8x16_t v_y[4]; + + float sumf = 0; + + for (int i = 0; i < nb; ++i) { + const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); + const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin); + + const int16x8_t v_ysumsl = vec_xl(0 , y[i].bsums); + const int16x8_t v_ysumsh = vec_xl(16, y[i].bsums); + const int16x8_t v_ysums = vec_padd_s16(v_ysumsl, v_ysumsh); + + memcpy(utmp, x[i].scales, 12); + utmp[3] = ((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4); + const uint32_t uaux = utmp[1] & kmask1; + utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4); + utmp[2] = uaux; + utmp[0] &= kmask1; + const uint8x16_t v_mins16 = vec_xl(0, (const uint8_t *)utmp); + const uint8x16_t v_mins8 = vec_perm(v_mins16, v_mins16, v_minsm); + const int16x8_t v_minsh = (int16x8_t)vec_unpackh(v_mins8); + + const int32x4_t v_minsho = vec_mulo(v_ysums, v_minsh); + const int32x4_t v_minshe = vec_mule(v_ysums, v_minsh); + const int32x4_t v_mins = vec_add(v_minsho, v_minshe); + const int32_t mins = v_mins[0] + v_mins[1] + v_mins[2] + v_mins[3]; + + const uint8_t * scales = (const uint8_t *)utmp; + const uint8_t * restrict x0l = x[i].qs; + const uint8_t * restrict x0h = x[i].qh; + const uint8_t * restrict y0 = y[i].qs; + + v_xh[0] = vec_xl(0 , x0h); + v_xh[1] = vec_xl(16, x0h); + + int32_t sumi = 0; + for (int j = 0; j < QK_K/64; ++j) { + v_xl[0] = vec_xl(0 , x0l); + v_xl[1] = vec_xl(16, x0l); + x0l += 32; + + v_y[0] = vec_xl(0 , y0); + v_y[1] = vec_xl(16, y0); + v_y[2] = vec_xl(32, y0); + v_y[3] = vec_xl(48, y0); + y0 += 64; + + q5h[0] = vec_sl(vec_and(v_1m, v_xh[0]), 4); + q5h[1] = vec_sl(vec_and(v_1m, v_xh[1]), 4); + q5h[2] = vec_sl(vec_and(v_2m, v_xh[0]), 3); + q5h[3] = vec_sl(vec_and(v_2m, v_xh[1]), 3); + v_xh[0] = vec_sr(v_xh[0], 2); + v_xh[1] = vec_sr(v_xh[1], 2); + + q5b[0] = (int8x16_t)vec_or(vec_and(v_xl[0], v_lm), q5h[0]); + q5b[1] = (int8x16_t)vec_or(vec_and(v_xl[1], v_lm), q5h[1]); + q5b[2] = (int8x16_t)vec_or(vec_sr(v_xl[0], 4), q5h[2]); + q5b[3] = (int8x16_t)vec_or(vec_sr(v_xl[1], 4), q5h[3]); + + int32x4_t sumi0 = ggml_vec_dot(ggml_vec_dot(v_z, q5b[0], v_y[0]), q5b[1], v_y[1]); + int32x4_t sumi1 = ggml_vec_dot(ggml_vec_dot(v_z, q5b[2], v_y[2]), q5b[3], v_y[3]); + + sumi += (sumi0[0] + sumi0[1] + sumi0[2] + sumi0[3]) * *scales++; + sumi += (sumi1[0] + sumi1[1] + sumi1[2] + sumi1[3]) * *scales++; + } + + sumf += d * sumi - dmin * mins; + } + + *s = sumf; #else const uint8_t * scales = (const uint8_t*)&utmp[0]; From c2794e8e87a172ec8782aaa02f6c5084ff603eb9 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Sun, 9 Feb 2025 20:31:19 +0800 Subject: [PATCH 44/56] ggml: fix wrong char*x16_t naming Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-impl.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-impl.h b/ggml/src/ggml-cpu/ggml-cpu-impl.h index 498a145f347f8..47e5ac81d905e 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-impl.h +++ b/ggml/src/ggml-cpu/ggml-cpu-impl.h @@ -394,8 +394,8 @@ inline static int32x4_t ggml_vdotq_s32(int32x4_t acc, int8x16_t a, int8x16_t b) #define vec_xor(a, b) ((a) ^ (b)) // Vector XOR #endif -typedef signed char char1x16_t __attribute__((vector_size(16))); -typedef unsigned char uchar1x16_t __attribute__((vector_size(16))); +typedef signed char char8x16_t __attribute__((vector_size(16))); +typedef unsigned char uchar8x16_t __attribute__((vector_size(16))); typedef int8_t int8x16_t __attribute__((vector_size(16))); typedef int16_t int16x8_t __attribute__((vector_size(16))); From 2606ddc9383cfb138c5e11b1e63146ced6c79cb7 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Sun, 9 Feb 2025 20:33:36 +0800 Subject: [PATCH 45/56] ggml: Q5_K y0 wrong signness Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index c274c79aacb80..6ce13a4d4a44d 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -6893,7 +6893,7 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r const uint8_t * scales = (const uint8_t *)utmp; const uint8_t * restrict x0l = x[i].qs; const uint8_t * restrict x0h = x[i].qh; - const uint8_t * restrict y0 = y[i].qs; + const int8_t * restrict y0 = y[i].qs; v_xh[0] = vec_xl(0 , x0h); v_xh[1] = vec_xl(16, x0h); From 809dac10ea04a14926ae68748cbcb6052b51189b Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Sun, 9 Feb 2025 20:34:57 +0800 Subject: [PATCH 46/56] ggml: fix Q5_K invalid uchar type Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 6ce13a4d4a44d..873a2eb0ee193 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -10899,8 +10899,8 @@ void ggml_vec_dot_iq4_nl_q8_0(int n, float * restrict s, size_t bs, const void * int8x16_t v_xl = (int8x16_t)vec_and(v_x, v_m); int8x16_t v_xh = (int8x16_t)vec_sr(v_x, 4); - v_xl = vec_perm(v_k, v_k, (uchar1x16_t)v_xl); - v_xh = vec_perm(v_k, v_k, (uchar1x16_t)v_xh); + v_xl = vec_perm(v_k, v_k, (uchar8x16_t)v_xl); + v_xh = vec_perm(v_k, v_k, (uchar8x16_t)v_xh); const int8x16_t v_yl = vec_xl(0 , y0->qs); const int8x16_t v_yh = vec_xl(QK8_0/2, y0->qs); From c8f9538b0e9f1d2c1afd484aee3420451a5db377 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Sun, 9 Feb 2025 20:35:53 +0800 Subject: [PATCH 47/56] ggml: fix Q5_K invalid uchar type Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 873a2eb0ee193..4da3eb3d8d5f1 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -11245,10 +11245,10 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * restrict s, size_t bs, const void * int8x16_t v_x1l = (int8x16_t)vec_and(v_x1, v_m); int8x16_t v_x1h = (int8x16_t)vec_sr(v_x1, 4); - v_x0l = vec_perm(v_k, v_k, (uchar1x16_t)v_x0l); - v_x0h = vec_perm(v_k, v_k, (uchar1x16_t)v_x0h); - v_x1l = vec_perm(v_k, v_k, (uchar1x16_t)v_x1l); - v_x1h = vec_perm(v_k, v_k, (uchar1x16_t)v_x1h); + v_x0l = vec_perm(v_k, v_k, (uchar8x16_t)v_x0l); + v_x0h = vec_perm(v_k, v_k, (uchar8x16_t)v_x0h); + v_x1l = vec_perm(v_k, v_k, (uchar8x16_t)v_x1l); + v_x1h = vec_perm(v_k, v_k, (uchar8x16_t)v_x1h); const int8x16_t v_y0 = vec_xl( 0, q8); const int8x16_t v_y1 = vec_xl(16, q8); From 3dd714460c17137782d2ce3925ab7ea58bd362fe Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Mon, 10 Feb 2025 14:29:46 +0800 Subject: [PATCH 48/56] ggml: s390x SIMD activation for Q4_K Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 71 +++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 4da3eb3d8d5f1..004db36d3fac1 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -6214,6 +6214,77 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * r ft_union fi; fi.i = __lsx_vpickve2gr_w(acc_m, 0); *s = hsum_float_8(acc) + fi.f ; +#elif defined(__VXE__) || defined(__VXE2__) + const uint8x16_t v_lm = vec_splat_u8(0x0F); + const int32x4_t v_z = vec_splat_s32(0); + + uint8x16_t v_x[2]; + int8x16_t v_xl[2]; + int8x16_t v_y[2]; + + float sumf = 0; + + for (int i = 0; i < nb; ++i) { + const float d = y[i].d * GGML_FP16_TO_FP32(x[i].d); + const float dmin = y[i].d * GGML_FP16_TO_FP32(x[i].dmin); + + const int16x8_t v_ysumsl = vec_xl(0 , y[i].bsums); + const int16x8_t v_ysumsh = vec_xl(16, y[i].bsums); + const int16x8_t v_ysums = vec_padd_s16(v_ysumsl, v_ysumsh); + + memcpy(utmp, x[i].scales, 12); + + uint32x4_t v_mins8 = { 0 }; + v_mins8 = vset_lane_u32(utmp[1] & kmask1, v_mins8, 0); + v_mins8 = vset_lane_u32(((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4), v_mins8, 1); + + utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4); + utmp[0] &= kmask1; + + const int16x8_t v_minsh = (int16x8_t)vec_unpackh((uint8x16_t)v_mins8); + + const int32x4_t v_minso = vec_mulo(v_ysums, v_minsh); + const int32x4_t v_minse = vec_mule(v_ysums, v_minsh); + const int32x4_t v_mins = v_minso + v_minse; + sumf -= dmin * (v_mins[0] + v_mins[1] + v_mins[2] + v_mins[3]); + + const uint8_t * scales = (const uint8_t *)utmp; + const uint8_t * restrict x0 = x[i].qs; + const int8_t * restrict y0 = y[i].qs; + + int32_t sumi1 = 0; + int32_t sumi2 = 0; + + for (int j = 0; j < QK_K/64; ++j) { + v_x[0] = vec_xl(0 , x0); + v_x[1] = vec_xl(16, x0); + x0 += 32; + + v_y[0] = vec_xl(0 , y0); + v_y[1] = vec_xl(16, y0); + y0 += 32; + + v_xl[0] = (int8x16_t)vec_and(v_x[0], v_lm); + v_xl[1] = (int8x16_t)vec_and(v_x[1], v_lm); + + const int32x4_t p1 = ggml_vec_dot(ggml_vec_dot(v_z, v_xl[0], v_y[0]), v_xl[1], v_y[1]); + sumi1 += (p1[0] + p1[1] + p1[2] + p1[3]) * scales[2*j+0]; + + v_y[0] = vec_xl(0 , y0); + v_y[1] = vec_xl(16, y0); + y0 += 32; + + v_xl[0] = (int8x16_t)vec_sr(v_x[0], 4); + v_xl[1] = (int8x16_t)vec_sr(v_x[1], 4); + + const int32x4_t p2 = ggml_vec_dot(ggml_vec_dot(v_z, v_xl[0], v_y[0]), v_xl[1], v_y[1]); + sumi2 += (p2[0] + p2[1] + p2[2] + p2[3]) * scales[2*j+1]; + } + + sumf += d * (sumi1 + sumi2); + } + + *s = sumf; #else const uint8_t * scales = (const uint8_t*)&utmp[0]; From 9b01b648e8fc05f50b9e537c3011b273d8f0b319 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Mon, 10 Feb 2025 14:30:51 +0800 Subject: [PATCH 49/56] ggml: fix Q4_K invalid vector intrinsics Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 004db36d3fac1..da7b8e3e3055c 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -6235,8 +6235,8 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * r memcpy(utmp, x[i].scales, 12); uint32x4_t v_mins8 = { 0 }; - v_mins8 = vset_lane_u32(utmp[1] & kmask1, v_mins8, 0); - v_mins8 = vset_lane_u32(((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4), v_mins8, 1); + v_mins8 = vec_insert(utmp[1] & kmask1, v_mins8, 0); + v_mins8 = vec_insert(((utmp[2] >> 4) & kmask2) | (((utmp[1] >> 6) & kmask3) << 4), v_mins8, 1); utmp[1] = (utmp[2] & kmask2) | (((utmp[0] >> 6) & kmask3) << 4); utmp[0] &= kmask1; From 84ee8b027cddb96dfed2de99d58ca25f56080037 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Sat, 15 Feb 2025 17:04:21 +0800 Subject: [PATCH 50/56] ggml: simplify ggml_padd_s16 compute kernel Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-impl.h | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-impl.h b/ggml/src/ggml-cpu/ggml-cpu-impl.h index 47e5ac81d905e..5cfa6420e13cc 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-impl.h +++ b/ggml/src/ggml-cpu/ggml-cpu-impl.h @@ -495,27 +495,12 @@ inline static int8x16_t ggml_vec_tbl(int8x16_t a, uint8x16_t b) { } inline static int16x8_t vec_padd_s16(int16x8_t a, int16x8_t b) { - const uchar8x16_t v_maske = { - 0x00, 0x01, 0x04, 0x05, 0x08, 0x09, 0x0C, 0x0D, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF - }; - - const uchar8x16_t v_masko = { - 0x02, 0x03, 0x06, 0x07, 0x0A, 0x0B, 0x0E, 0x0F, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF - }; - - const uchar8x16_t v_maskj = { - 0, 1, 2, 3, 4, 5, 6, 7, - 16, 17, 18, 19, 20, 21, 22, 23 - }; - - const int16x8_t pa = vec_add(vec_perm(a, a, v_maske), - vec_perm(a, a, v_masko)); - const int16x8_t pb = vec_add(vec_perm(b, b, v_maske), - vec_perm(b, b, v_masko)); - - return vec_perm(pa, pb, v_maskj); + const uchar8x16_t v_maske = { 0, 1, 4, 5, 8, 9, 12, 13, + 16, 17, 20, 21, 24, 25, 28, 29 }; + + const int16x8_t v_abo = vec_pack((int32x4_t)a, (int32x4_t)b); + const int16x8_t v_abe = vec_perm(a, b, v_maske); + return v_abo + v_abe; } inline static int32x4_t ggml_vec_dot(int32x4_t acc, int8x16_t a, int8x16_t b) { From 8ced2ab396e2fd5274120e404b0da078892a7d10 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Sat, 22 Feb 2025 15:47:44 +0800 Subject: [PATCH 51/56] ggml: correct ggml-cpu vxe wording Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index aeb52843dcfe9..13e9edb80e618 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -321,7 +321,7 @@ function(ggml_add_cpu_backend_variant_impl tag_name) list(APPEND ARCH_FLAGS -march=z16 -mtune=z16) else() message(STATUS "Unknown target") - message(WARNING "Unknown target. If you are compiling for z15 and earlier, you might have to add -DGGML_VXE=OFF.") + message(WARNING "Unknown target. If you are compiling for z14 and earlier, you might have to add -DGGML_VXE=OFF.") list(APPEND ARCH_FLAGS -march=native -mtune=native) endif() From 5796caf71a7d3a0c9a957894631e8e9cfd2227c6 Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Sat, 22 Feb 2025 15:50:30 +0800 Subject: [PATCH 52/56] ggml: change ggml_aligned_malloc alignment to 256 256 is the cache line size for s390x platforms Signed-off-by: Aaron Teo --- ggml/src/ggml.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 2bbe5f48257b2..bccde7c1b8524 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -236,7 +236,11 @@ void ggml_log_callback_default(enum ggml_log_level level, const char * text, voi void * ggml_aligned_malloc(size_t size) { +#if defined(__s390x__) + const int alignment = 256; +#else const int alignment = 64; +#endif #if defined(_MSC_VER) || defined(__MINGW32__) return _aligned_malloc(size, alignment); From b4b22140f26e99f2f4085fb953895640cb45c337 Mon Sep 17 00:00:00 2001 From: Jinyang He Date: Fri, 7 Feb 2025 15:38:31 +0800 Subject: [PATCH 53/56] ggml: resolve pr merge via cherry-pick 225bbbf Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-impl.h | 18 +++++--------- ggml/src/ggml-cpu/ggml-cpu-quants.c | 37 ++++++----------------------- ggml/src/ggml-cpu/ggml-cpu.c | 24 +++++++------------ 3 files changed, 22 insertions(+), 57 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-impl.h b/ggml/src/ggml-cpu/ggml-cpu-impl.h index 5cfa6420e13cc..7f7d210cbe5d5 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-impl.h +++ b/ggml/src/ggml-cpu/ggml-cpu-impl.h @@ -511,21 +511,15 @@ inline static int32x4_t ggml_vec_dot(int32x4_t acc, int8x16_t a, int8x16_t b) { #endif #if defined(__loongarch_asx) - -typedef union { - int32_t i; - float f; -} ft_union; - /* float type data load instructions */ -static __m128 __lsx_vreplfr2vr_s(float val) { - ft_union fi_tmpval = {.f = val}; - return (__m128)__lsx_vreplgr2vr_w(fi_tmpval.i); +static __m128 __lsx_vreplfr2vr_s(const float val) { + v4f32 res = {val, val, val, val}; + return (__m128)res; } -static __m256 __lasx_xvreplfr2vr_s(float val) { - ft_union fi_tmpval = {.f = val}; - return (__m256)__lasx_xvreplgr2vr_w(fi_tmpval.i); +static __m256 __lasx_xvreplfr2vr_s(const float val) { + v8f32 res = {val, val, val, val, val, val, val, val}; + return (__m256)res; } #endif diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index da7b8e3e3055c..4fa6400b81376 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -434,30 +434,15 @@ static __m256i lasx_shuffle_b(__m256i a, __m256i b) { } static __m256i lasx_extu8_16(__m128i a) { - __m128i zero = __lsx_vldi(0); - __m128i vlo = __lsx_vilvl_b(zero, a); - __m128i vhi = __lsx_vilvh_b(zero, a); - return lasx_set_q(vhi, vlo); + return __lasx_vext2xv_hu_bu(____m256i(a)); } static __m256i lasx_ext8_16(__m128i a) { - __m128i sign = __lsx_vslti_b(a, 0); - __m128i vlo = __lsx_vilvl_b(sign, a); - __m128i vhi = __lsx_vilvh_b(sign, a); - return lasx_set_q(vhi, vlo); + return __lasx_vext2xv_h_b(____m256i(a)); } static __m256i lasx_ext16_32(__m128i a) { - __m256i tmp1; - tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 0), 0); - tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 1), 1); - tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 2), 2); - tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 3), 3); - tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 4), 4); - tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 5), 5); - tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 6), 6); - tmp1 = __lasx_xvinsgr2vr_w(tmp1, __lsx_vpickve2gr_h(a, 7), 7); - return tmp1; + return __lasx_vext2xv_w_h(____m256i(a)); } static __m128i lasx_extracti128( __m256i a, int pos) { @@ -580,12 +565,10 @@ static inline __m128i mul_sum_i8_pairs(const __m128i x, const __m128i y) { // horizontally add 8 floats static inline float hsum_float_8(const __m256 x) { __m128 res = lasx_extractf128(x, 1); - ft_union tmp; res = __lsx_vfadd_s(res, lasx_extractf128(x, 0)); res = __lsx_vfadd_s(res, (__m128)__lsx_vpickod_d((__m128i)res, (__m128i)res)); res = __lsx_vfadd_s(res, (__m128)__lsx_vinsgr2vr_w(__lsx_vldi(0), __lsx_vpickve2gr_w(res, 1), 0)); - tmp.i = __lsx_vpickve2gr_w(res, 0); - return tmp.f; + return ((v4f32)res)[0]; } // horizontally add 8 int32_t @@ -927,7 +910,6 @@ void quantize_row_q8_0(const float * restrict x, void * restrict vy, int64_t k) #elif defined(__loongarch_asx) for (int i = 0; i < nb; i++) { - ft_union fi; __m256 v0 = (__m256)__lasx_xvld( x , 0); __m256 v1 = (__m256)__lasx_xvld( x , 32); __m256 v2 = (__m256)__lasx_xvld( x , 64); @@ -945,8 +927,7 @@ void quantize_row_q8_0(const float * restrict x, void * restrict vy, int64_t k) max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vpickod_d((__m128i) max4, (__m128i)max4 ) ); __m128 tmp = max4; max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vinsgr2vr_w(tmp, __lsx_vpickve2gr_w( max4, 1 ), 0 )); - fi.i = __lsx_vpickve2gr_w( (__m128i)max4, 0 ); - const float max_scalar = fi.f; + const float max_scalar = ((v4f32)max4)[0]; // Quantize these floats const float d = max_scalar / 127.f; @@ -1283,7 +1264,6 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int64_t k) #elif defined(__loongarch_asx) for (int i = 0; i < nb; i++) { - ft_union ft; __m256 v0 = (__m256)__lasx_xvld( x , 0 ); __m256 v1 = (__m256)__lasx_xvld( x , 32 ); __m256 v2 = (__m256)__lasx_xvld( x , 64 ); @@ -1301,8 +1281,7 @@ void quantize_row_q8_1(const float * restrict x, void * restrict vy, int64_t k) max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vpickod_d((__m128i) max4, (__m128i)max4 ) ); __m128 tmp = max4; max4 = __lsx_vfmax_s( max4, (__m128)__lsx_vextrins_w((__m128i)tmp, (__m128i)max4, 0x10 )); - ft.i = __lsx_vpickve2gr_w( (__m128i)max4, 0 ); - const float max_scalar = ft.f; + const float max_scalar = ((v4f32)max4)[0]; // Quantize these floats const float d = max_scalar / 127.f; @@ -6211,9 +6190,7 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * r acc_m = __lsx_vfadd_s(acc_m, (__m128)tmp1); - ft_union fi; - fi.i = __lsx_vpickve2gr_w(acc_m, 0); - *s = hsum_float_8(acc) + fi.f ; + *s = hsum_float_8(acc) + ((v4f32)acc_m)[0]; #elif defined(__VXE__) || defined(__VXE2__) const uint8x16_t v_lm = vec_splat_u8(0x0F); const int32x4_t v_z = vec_splat_s32(0); diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index aa89764e959ae..8406dd2f091c3 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -1080,29 +1080,23 @@ do { \ #define GGML_F16_STEP 32 #define GGML_F16_EPR 8 -// F16 arithmetic is not supported by AVX, so we use F32 instead +// F16 arithmetic is not supported by LASX, so we use F32 instead #define GGML_F32Cx8 __m256 #define GGML_F32Cx8_ZERO (__m256)__lasx_xvldi(0) #define GGML_F32Cx8_SET1(x) (__m256)__lasx_xvreplgr2vr_w((x)) static inline __m256 __lasx_f32cx8_load(const ggml_fp16_t * x) { - float tmp[8]; - - for (int i = 0; i < 8; i++) { - tmp[i] = GGML_FP16_TO_FP32(x[i]); - } - - return (__m256)__lasx_xvld(tmp, 0); + __m256i a; + memcpy(&a, x, sizeof(ggml_fp16_t) * 8); + a = __lasx_xvpermi_d(a, 0 | (1 << 4)); + return __lasx_xvfcvtl_s_h(a); } -static inline void __lasx_f32cx8_store(ggml_fp16_t * x, __m256 y) { - float arr[8]; - __lasx_xvst(y, arr, 0); - - for (int i = 0; i < 8; i++) { - x[i] = GGML_FP32_TO_FP16(arr[i]); - } +static inline void __lasx_f32cx8_store(ggml_fp16_t * x, __m256 y) { + __m256i a = __lasx_xvfcvt_h_s(y, y); + a = __lasx_xvpermi_d(a, 0 | (2 << 2)); + memcpy(x, &a, sizeof(ggml_fp16_t) * 8); } #define GGML_F32Cx8_LOAD(x) __lasx_f32cx8_load(x) #define GGML_F32Cx8_STORE(x, y) __lasx_f32cx8_store(x, y) From cfc2603376900fee0ed2f826a03e18bbdba592f8 Mon Sep 17 00:00:00 2001 From: junchao-zhao <68935141+junchao-loongson@users.noreply.github.com> Date: Thu, 6 Feb 2025 17:20:00 +0800 Subject: [PATCH 54/56] ggml : fix LoongArch compile error with 128-bit SIMD (#11701) --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 169 +++++++++++++++------------- 1 file changed, 91 insertions(+), 78 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 4fa6400b81376..6446881975cd0 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -297,6 +297,90 @@ static const uint64_t table_b2b_0[1 << 8] = { B8(00, 10) }; // ( b) << 4 static const uint64_t table_b2b_1[1 << 8] = { B8(10, 00) }; // (!b) << 4 #endif +#if defined(__loongarch_sx) + +static __m128i lsx_packs_w(__m128i a, __m128i b) { + __m128i tmp, tmp1; + tmp = __lsx_vsat_w(a, 15); + tmp1 = __lsx_vsat_w(b, 15); + return __lsx_vpickev_h(tmp1, tmp); +} + +static __m128i lsx_packs_h(__m128i a, __m128i b) { + __m128i tmp, tmp1; + tmp = __lsx_vsat_h(a, 7); + tmp1 = __lsx_vsat_h(b, 7); + return __lsx_vpickev_b(tmp1, tmp); +} + +static __m128i lsx_packus_h(__m128i a, __m128i b) { + __m128i tmp, tmp1; + tmp = __lsx_vsat_hu(a, 7); + tmp1 = __lsx_vsat_hu(b, 7); + return __lsx_vpickev_b(tmp1, tmp); +} + +static __m128i lsx_maddubs_h(__m128i a, __m128i b) { + __m128i tmp1, tmp2; + tmp1 = __lsx_vmulwev_h_b(a, b); + tmp2 = __lsx_vmulwod_h_b(a, b); + return __lsx_vsadd_h(tmp1, tmp2); +} + +static __m128i lsx_madd_h(__m128i a, __m128i b) { + __m128i tmp1, tmp2; + tmp1 = __lsx_vmulwev_w_h(a, b); + tmp2 = __lsx_vmulwod_w_h(a, b); + return __lsx_vadd_w(tmp1, tmp2); +} + +static __m128i lsx_set_w(int32_t a, int32_t b, int32_t c, int32_t d) { + v4i32 __ret = {d, c, b, a}; + return (__m128i)__ret; +} + +static __m128i lsx_shuffle_b(__m128i a, __m128i b) { + __m128i mask_f, zero, tmp0, tmp2, mask; + int f = 0x8f; + mask_f = __lsx_vreplgr2vr_b(f); + zero = __lsx_vldi(0); + tmp0 = __lsx_vand_v(b, mask_f); // get mask with low 4 bit and sign bits + tmp0 = __lsx_vori_b(tmp0, 0x10); // make each mask or with 0x10 prepare for positive + mask = __lsx_vsle_b(zero, tmp0); // if mask >= 0, set mask + tmp2 = __lsx_vand_v(tmp0, mask); // maskout the in2 < ones + return __lsx_vshuf_b(a, zero, tmp2); +} + +static __m128i lsx_hadd_h(__m128i a, __m128i b) { + __m128i tmp1 = __lsx_vpickev_h(b, a); + __m128i tmp2 = __lsx_vpickod_h(b, a); + return __lsx_vadd_h(tmp1, tmp2); +} + +static __m128i lsx_hadd_w(__m128i a, __m128i b) { + __m128i tmp1 = __lsx_vpickev_w(b, a); + __m128i tmp2 = __lsx_vpickod_w(b, a); + return __lsx_vadd_w(tmp1, tmp2); +} + +static __m128 lsx_hadd_s(__m128 a, __m128 b) { + __m128 tmp1 = (__m128)__lsx_vpickev_w((__m128i)b, (__m128i)a); + __m128 tmp2 = (__m128)__lsx_vpickod_w((__m128i)b, (__m128i)a); + + return __lsx_vfadd_s(tmp1, tmp2); +} + +static inline float hsum_float_4x4(const __m128 a, const __m128 b, const __m128 c, const __m128 d) { + __m128 res_0 =lsx_hadd_s(a, b); + __m128 res_1 =lsx_hadd_s(c, d); + __m128 res =lsx_hadd_s(res_0, res_1); + res =lsx_hadd_s(res, res); + res =lsx_hadd_s(res, res); + + return ((v4f32)res)[0]; +} +#endif + #if defined(__loongarch_asx) #ifdef __clang__ @@ -395,11 +479,6 @@ static __m256i lasx_set_w(int e7, int e6, int e5, int e4, int e3, int e2, int e1 return (__m256i)__ret; } -static __m128i lsx_set_w(int32_t a, int32_t b, int32_t c, int32_t d) { - v4i32 __ret = {d, c, b, a}; - return (__m128i)__ret; -} - static __m256i lasx_set_d(int64_t a, int64_t b, int64_t c, int64_t d) { v4i64 __ret = {d, c, b, a}; return (__m256i)__ret; @@ -409,18 +488,6 @@ static __m256i lasx_insertf128( __m128i x, __m128i y) { return lasx_set_q(x, y); } -static __m128i lsx_shuffle_b(__m128i a, __m128i b) { - __m128i mask_f, zero, tmp0, tmp2, mask; - int f = 0x8f; - mask_f = __lsx_vreplgr2vr_b(f); - zero = __lsx_vldi(0); - tmp0 = __lsx_vand_v(b, mask_f); // get mask with low 4 bit and sign bits - tmp0 = __lsx_vori_b(tmp0, 0x10); // make each mask or with 0x10 prepare for positive - mask = __lsx_vsle_b(zero, tmp0); // if mask >= 0, set mask - tmp2 = __lsx_vand_v(tmp0, mask); // maskout the in2 < ones - return __lsx_vshuf_b(a, zero, tmp2); -} - static __m256i lasx_shuffle_b(__m256i a, __m256i b) { __m256i mask_f, zero, tmp0, tmp2, mask; int f = 0x8f; @@ -467,25 +534,6 @@ static __m128 lasx_extractf128( __m256 a, int pos) { return ret; } -static __m128i lsx_hadd_h(__m128i a, __m128i b) { - __m128i tmp1 = __lsx_vpickev_h(b, a); - __m128i tmp2 = __lsx_vpickod_h(b, a); - return __lsx_vadd_h(tmp1, tmp2); -} - -static __m128i lsx_hadd_w(__m128i a, __m128i b) { - __m128i tmp1 = __lsx_vpickev_w(b, a); - __m128i tmp2 = __lsx_vpickod_w(b, a); - return __lsx_vadd_w(tmp1, tmp2); -} - -static __m128 lsx_hadd_s(__m128 a, __m128 b) { - __m128 tmp1 = (__m128)__lsx_vpickev_w((__m128i)b, (__m128i)a); - __m128 tmp2 = (__m128)__lsx_vpickod_w((__m128i)b, (__m128i)a); - - return __lsx_vfadd_s(tmp1, tmp2); -} - static __m256i lasx_maddubs_h(__m256i a, __m256i b) { __m256i tmp1, tmp2; tmp1 = __lasx_xvmulwev_h_b(a, b); @@ -514,42 +562,6 @@ static __m256i lasx_packs_h(__m256i a, __m256i b) { return __lasx_xvpickev_b(tmp1, tmp); } -static __m128i lsx_packs_w(__m128i a, __m128i b) { - __m128i tmp, tmp1; - tmp = __lsx_vsat_w(a, 15); - tmp1 = __lsx_vsat_w(b, 15); - return __lsx_vpickev_h(tmp1, tmp); -} - -static __m128i lsx_packs_h(__m128i a, __m128i b) { - __m128i tmp, tmp1; - tmp = __lsx_vsat_h(a, 7); - tmp1 = __lsx_vsat_h(b, 7); - return __lsx_vpickev_b(tmp1, tmp); -} - -static __m128i lsx_packus_h(__m128i a, __m128i b) { - __m128i tmp, tmp1; - tmp = __lsx_vsat_hu(a, 7); - tmp1 = __lsx_vsat_hu(b, 7); - return __lsx_vpickev_b(tmp1, tmp); -} - - -static __m128i lsx_maddubs_h(__m128i a, __m128i b) { - __m128i tmp1, tmp2; - tmp1 = __lsx_vmulwev_h_b(a, b); - tmp2 = __lsx_vmulwod_h_b(a, b); - return __lsx_vsadd_h(tmp1, tmp2); -} - -static __m128i lsx_madd_h(__m128i a, __m128i b) { - __m128i tmp1, tmp2; - tmp1 = __lsx_vmulwev_w_h(a, b); - tmp2 = __lsx_vmulwod_w_h(a, b); - return __lsx_vadd_w(tmp1, tmp2); -} - // multiply int8_t, add results pairwise twice static inline __m128i mul_sum_i8_pairs(const __m128i x, const __m128i y) { // Get absolute values of x vectors @@ -2281,21 +2293,22 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * r } sumf = hsum_float_8(acc); + #elif defined(__loongarch_sx) // set constants const __m128i low_mask = __lsx_vreplgr2vr_b(0xF); const __m128i off = __lsx_vreplgr2vr_b(8); // Initialize accumulator with zeros - __m128 acc_0 = __lsx_vldi(0); - __m128 acc_1 = __lsx_vldi(0); - __m128 acc_2 = __lsx_vldi(0); - __m128 acc_3 = __lsx_vldi(0); + __m128 acc_0 = (__m128)__lsx_vldi(0); + __m128 acc_1 = (__m128)__lsx_vldi(0); + __m128 acc_2 = (__m128)__lsx_vldi(0); + __m128 acc_3 = (__m128)__lsx_vldi(0); for (; ib + 1 < nb; ib += 2) { // Compute combined scale for the block 0 and 1 - const __m128 d_0_1 = __lsx_vreplgr2vr_w( GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d) ); + const __m128 d_0_1 = (__m128)__lsx_vreplgr2vr_w( GGML_FP16_TO_FP32(x[ib].d) * GGML_FP16_TO_FP32(y[ib].d) ); const __m128i tmp_0_1 = __lsx_vld((const __m128i *)x[ib].qs, 0); @@ -2313,7 +2326,7 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * restrict s, size_t bs, const void * r //_mm_prefetch(&y[ib] + 2 * sizeof(block_q8_0), _MM_HINT_T0); // Compute combined scale for the block 2 and 3 - const __m128 d_2_3 = __lsx_vreplgr2vr_w( GGML_FP16_TO_FP32(x[ib + 1].d) * GGML_FP16_TO_FP32(y[ib + 1].d) ); + const __m128 d_2_3 = (__m128)__lsx_vreplgr2vr_w( GGML_FP16_TO_FP32(x[ib + 1].d) * GGML_FP16_TO_FP32(y[ib + 1].d) ); const __m128i tmp_2_3 = __lsx_vld((const __m128i *)x[ib + 1].qs, 0); From f263ec38ec32ea3fa9dcd682b1d1b2443523f7b9 Mon Sep 17 00:00:00 2001 From: Jinyang He Date: Fri, 14 Feb 2025 16:54:27 +0800 Subject: [PATCH 55/56] ggml: resolve pr merge via cherry-pick 4571953 Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/ggml-cpu-quants.c | 362 +++++++++++----------------- 1 file changed, 141 insertions(+), 221 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu-quants.c b/ggml/src/ggml-cpu/ggml-cpu-quants.c index 6446881975cd0..5fe5b4b5039eb 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-quants.c +++ b/ggml/src/ggml-cpu/ggml-cpu-quants.c @@ -562,6 +562,41 @@ static __m256i lasx_packs_h(__m256i a, __m256i b) { return __lasx_xvpickev_b(tmp1, tmp); } +static inline __m256i lasx_madd_h_b(__m256i a, __m256i b) { + __m256i tmp1, tmp2; + tmp1 = __lasx_xvmulwev_h_b(a, b); + tmp2 = __lasx_xvmulwod_h_b(a, b); + return __lasx_xvadd_h(tmp1, tmp2); +} + +static inline __m256i lasx_xvrepl128vei_h(__m256i a, const unsigned int b) { + switch (b) { + case 0: return __lasx_xvrepl128vei_h(a, 0); + case 1: return __lasx_xvrepl128vei_h(a, 1); + case 2: return __lasx_xvrepl128vei_h(a, 2); + case 3: return __lasx_xvrepl128vei_h(a, 3); + case 4: return __lasx_xvrepl128vei_h(a, 4); + case 5: return __lasx_xvrepl128vei_h(a, 5); + case 6: return __lasx_xvrepl128vei_h(a, 6); + case 7: return __lasx_xvrepl128vei_h(a, 7); + default: __builtin_unreachable(); + } +} + +static inline __m256i lasx_xvandi_b_bit(__m256i a, const unsigned int b) { + switch (b) { + case 0: return __lasx_xvandi_b(a, 1 << 0); + case 1: return __lasx_xvandi_b(a, 1 << 1); + case 2: return __lasx_xvandi_b(a, 1 << 2); + case 3: return __lasx_xvandi_b(a, 1 << 3); + case 4: return __lasx_xvandi_b(a, 1 << 4); + case 5: return __lasx_xvandi_b(a, 1 << 5); + case 6: return __lasx_xvandi_b(a, 1 << 6); + case 7: return __lasx_xvandi_b(a, 1 << 7); + default: __builtin_unreachable(); + } +} + // multiply int8_t, add results pairwise twice static inline __m128i mul_sum_i8_pairs(const __m128i x, const __m128i y) { // Get absolute values of x vectors @@ -656,13 +691,8 @@ static inline __m256 mul_sum_us8_pairs_float(const __m256i ax, const __m256i sy) // multiply int8_t, add results pairwise twice and return as float vector static inline __m256 mul_sum_i8_pairs_float(const __m256i x, const __m256i y) { - - // Get absolute values of x vectors - const __m256i ax = __lasx_xvsigncov_b(x, x); - // Sign the values of the y vectors - const __m256i sy = __lasx_xvsigncov_b(x, y); - - return mul_sum_us8_pairs_float(ax, sy); + const __m256i dot = lasx_madd_h_b(x, y); + return sum_i16_pairs_float(dot); } static inline __m128i packNibbles( __m256i bytes ) { @@ -4809,9 +4839,6 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * r #elif defined __loongarch_asx - const __m256i m3 = __lasx_xvreplgr2vr_b(3); - const __m128i m4 = __lsx_vreplgr2vr_b(0xF); - __m256 acc = (__m256)__lasx_xvldi(0); for (int i = 0; i < nb; ++i) { @@ -4822,18 +4849,15 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * r const uint8_t * restrict q2 = x[i].qs; const int8_t * restrict q8 = y[i].qs; - const __m128i mins_and_scales = __lsx_vld((const __m128i*)x[i].scales, 0); - const __m128i scales8 = __lsx_vand_v(mins_and_scales, m4); - const __m128i mins8 = __lsx_vand_v(__lsx_vsrli_h(mins_and_scales, 4), m4); - const __m256i mins = lasx_ext8_16(mins8); + const __m128i mins_and_scales128 = __lsx_vld((const __m128i*)x[i].scales, 0); + const __m128i scales128 = __lsx_vandi_b(mins_and_scales128, 0xf); + const __m256i mins = lasx_ext8_16(__lsx_vsrli_b(mins_and_scales128, 4)); const __m256i prod = lasx_madd_h(mins, __lasx_xvld((const __m256i*)y[i].bsums, 0)); acc = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(dmin), __lasx_xvffint_s_w(prod), acc); - const __m256i all_scales = lasx_ext8_16(scales8); - const __m128i l_scales = lasx_extracti128(all_scales, 0); - const __m128i h_scales = lasx_extracti128(all_scales, 1); - const __m256i scales[2] = {lasx_insertf128(l_scales, l_scales), lasx_insertf128(h_scales, h_scales)}; + const v16i8 shuffle_mask = {0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15}; + const __m256i scales_shuffled = lasx_ext8_16(__lsx_vshuf_b(scales128, scales128, (__m128i)shuffle_mask)); __m256i sumi = __lasx_xvldi(0); @@ -4846,20 +4870,20 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * restrict s, size_t bs, const void * r const __m256i q8_2 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; const __m256i q8_3 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; - const __m256i q2_0 = __lasx_xvand_v(q2bits, m3); - const __m256i q2_1 = __lasx_xvand_v(__lasx_xvsrli_h(q2bits, 2), m3); - const __m256i q2_2 = __lasx_xvand_v(__lasx_xvsrli_h(q2bits, 4), m3); - const __m256i q2_3 = __lasx_xvand_v(__lasx_xvsrli_h(q2bits, 6), m3); + const __m256i q2_0 = __lasx_xvandi_b(q2bits, 3); + const __m256i q2_1 = __lasx_xvandi_b(__lasx_xvsrli_b(q2bits, 2), 3); + const __m256i q2_2 = __lasx_xvandi_b(__lasx_xvsrli_b(q2bits, 4), 3); + const __m256i q2_3 = __lasx_xvsrli_b(q2bits, 6); - __m256i p0 = lasx_maddubs_h(q2_0, q8_0); - __m256i p1 = lasx_maddubs_h(q2_1, q8_1); - __m256i p2 = lasx_maddubs_h(q2_2, q8_2); - __m256i p3 = lasx_maddubs_h(q2_3, q8_3); + __m256i p0 = lasx_madd_h_b(q2_0, q8_0); + __m256i p1 = lasx_madd_h_b(q2_1, q8_1); + __m256i p2 = lasx_madd_h_b(q2_2, q8_2); + __m256i p3 = lasx_madd_h_b(q2_3, q8_3); - p0 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(0)), p0); - p1 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(1)), p1); - p2 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(2)), p2); - p3 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(3)), p3); + p0 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 0), p0); + p1 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 1), p1); + p2 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 2), p2); + p3 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 3), p3); p0 = __lasx_xvadd_w(p0, p1); p2 = __lasx_xvadd_w(p2, p3); @@ -5527,8 +5551,6 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * r #elif defined __loongarch_asx - const __m256i m3 = __lasx_xvreplgr2vr_b(3); - const __m256i mone = __lasx_xvreplgr2vr_b(1); const __m128i m32 = __lsx_vreplgr2vr_b(32); __m256 acc = (__m256)__lasx_xvldi(0); @@ -5548,10 +5570,9 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * r (aux[1] & kmask2) | (((aux[2] >> 2) & kmask1) << 4), (aux[0] & kmask2) | (((aux[2] >> 0) & kmask1) << 4)); scales128 = __lsx_vsub_b(scales128, m32); - const __m256i all_scales = lasx_ext8_16(scales128); - const __m128i l_scales = lasx_extracti128(all_scales, 0); - const __m128i h_scales = lasx_extracti128(all_scales, 1); - const __m256i scales[2] = {lasx_insertf128(l_scales, l_scales), lasx_insertf128(h_scales, h_scales)}; + + const v16i8 shuffle_mask = {0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15}; + const __m256i scales_shuffled = lasx_ext8_16(__lsx_vshuf_b(scales128, scales128, (__m128i)shuffle_mask)); // high bit const __m256i hbits = __lasx_xvld((const __m256i*)x[i].hmask, 0); @@ -5559,35 +5580,23 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * r // integer accumulator __m256i sumi = __lasx_xvldi(0); - int bit = 0; - int is = 0; - __m256i xvbit; - - for (int j = 0; j < QK_K/128; ++j) { // load low 2 bits const __m256i q3bits = __lasx_xvld((const __m256i*)q3, 0); q3 += 32; - xvbit = __lasx_xvreplgr2vr_h(bit); // prepare low and high bits - const __m256i q3l_0 = __lasx_xvand_v(q3bits, m3); - const __m256i q3h_0 = __lasx_xvslli_h(__lasx_xvsrl_h(__lasx_xvandn_v(hbits, __lasx_xvsll_h(mone, xvbit)), xvbit), 2); - ++bit; - - xvbit = __lasx_xvreplgr2vr_h(bit); - const __m256i q3l_1 = __lasx_xvand_v(__lasx_xvsrli_h(q3bits, 2), m3); - const __m256i q3h_1 = __lasx_xvslli_h(__lasx_xvsrl_h(__lasx_xvandn_v(hbits, __lasx_xvsll_h(mone, xvbit)), xvbit), 2); - ++bit; - - xvbit = __lasx_xvreplgr2vr_h(bit); - const __m256i q3l_2 = __lasx_xvand_v(__lasx_xvsrli_h(q3bits, 4), m3); - const __m256i q3h_2 = __lasx_xvslli_h(__lasx_xvsrl_h(__lasx_xvandn_v(hbits, __lasx_xvsll_h(mone, xvbit)), xvbit), 2); - ++bit; - - xvbit = __lasx_xvreplgr2vr_h(bit); - const __m256i q3l_3 = __lasx_xvand_v(__lasx_xvsrli_h(q3bits, 6), m3); - const __m256i q3h_3 = __lasx_xvslli_h(__lasx_xvsrl_h(__lasx_xvandn_v(hbits, __lasx_xvsll_h(mone, xvbit)), xvbit), 2); - ++bit; + const __m256i q3l_0 = __lasx_xvandi_b(q3bits, 3); + const __m256i q3l_1 = __lasx_xvandi_b(__lasx_xvsrli_b(q3bits, 2), 3); + const __m256i q3l_2 = __lasx_xvandi_b(__lasx_xvsrli_b(q3bits, 4), 3); + const __m256i q3l_3 = __lasx_xvsrli_b(q3bits, 6); + const __m256i q3h_0 = __lasx_xvslli_b(__lasx_xvseqi_b(lasx_xvandi_b_bit(hbits, 4 * j + 0), 0), 2); + const __m256i q3h_1 = __lasx_xvslli_b(__lasx_xvseqi_b(lasx_xvandi_b_bit(hbits, 4 * j + 1), 0), 2); + const __m256i q3h_2 = __lasx_xvslli_b(__lasx_xvseqi_b(lasx_xvandi_b_bit(hbits, 4 * j + 2), 0), 2); + const __m256i q3h_3 = __lasx_xvslli_b(__lasx_xvseqi_b(lasx_xvandi_b_bit(hbits, 4 * j + 3), 0), 2); + const __m256i q3_0 = __lasx_xvor_v(q3h_0, q3l_0); + const __m256i q3_1 = __lasx_xvor_v(q3h_1, q3l_1); + const __m256i q3_2 = __lasx_xvor_v(q3h_2, q3l_2); + const __m256i q3_3 = __lasx_xvor_v(q3h_3, q3l_3); // load Q8 quants const __m256i q8_0 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; @@ -5595,29 +5604,16 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * r const __m256i q8_2 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; const __m256i q8_3 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; - // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we can use lasx_maddubs_h, - // and then subtract. The high bit part has the 2 already subtracted (and so, it is zero if the high bit was not set, - // and 2 if the high bit was set) - __m256i q8s_0 = lasx_maddubs_h(q3h_0, q8_0); - __m256i q8s_1 = lasx_maddubs_h(q3h_1, q8_1); - __m256i q8s_2 = lasx_maddubs_h(q3h_2, q8_2); - __m256i q8s_3 = lasx_maddubs_h(q3h_3, q8_3); - - __m256i p16_0 = lasx_maddubs_h(q3l_0, q8_0); - __m256i p16_1 = lasx_maddubs_h(q3l_1, q8_1); - __m256i p16_2 = lasx_maddubs_h(q3l_2, q8_2); - __m256i p16_3 = lasx_maddubs_h(q3l_3, q8_3); - - p16_0 = __lasx_xvsub_h(p16_0, q8s_0); - p16_1 = __lasx_xvsub_h(p16_1, q8s_1); - p16_2 = __lasx_xvsub_h(p16_2, q8s_2); - p16_3 = __lasx_xvsub_h(p16_3, q8s_3); + __m256i p16_0 = lasx_madd_h_b(q8_0, q3_0); + __m256i p16_1 = lasx_madd_h_b(q8_1, q3_1); + __m256i p16_2 = lasx_madd_h_b(q8_2, q3_2); + __m256i p16_3 = lasx_madd_h_b(q8_3, q3_3); // multiply with scales - p16_0 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(is + 0)), p16_0); - p16_1 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(is + 1)), p16_1); - p16_2 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(is + 2)), p16_2); - p16_3 = lasx_madd_h(lasx_shuffle_b(scales[j], get_scale_shuffle_q3k(is + 3)), p16_3); + p16_0 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 0), p16_0); + p16_1 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 1), p16_1); + p16_2 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 2), p16_2); + p16_3 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 3), p16_3); // accumulate p16_0 = __lasx_xvadd_w(p16_0, p16_1); @@ -5625,7 +5621,7 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * restrict s, size_t bs, const void * r sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_0, p16_2)); } // multiply with block scale and accumulate - acc = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), acc);//FIXME + acc = __lasx_xvfmadd_s(__lasx_xvreplfr2vr_s(d), __lasx_xvffint_s_w(sumi), acc); } *s = hsum_float_8(acc); @@ -6136,11 +6132,6 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * r *s = vec_extract(vsumf0, 0); #elif defined __loongarch_asx - GGML_UNUSED(kmask1); - GGML_UNUSED(kmask2); - GGML_UNUSED(kmask3); - - const __m256i m4 = __lasx_xvreplgr2vr_b(0xF); __m256 acc = (__m256)__lasx_xvldi(0); __m128 acc_m = (__m128)__lsx_vldi(0); @@ -6160,33 +6151,34 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * restrict s, size_t bs, const void * r const uint8_t * restrict q4 = x[i].qs; const int8_t * restrict q8 = y[i].qs; - const __m256i mins_and_scales = lasx_extu8_16(lsx_set_w(utmp[3], utmp[2], utmp[1], utmp[0])); + const __m128i mins_and_scales128 = lsx_set_w(utmp[3], utmp[2], utmp[1], utmp[0]); + const __m128i mins128 = __lsx_vexth_h_b(mins_and_scales128); + const __m128i scales128 = __lsx_vsllwil_h_b(mins_and_scales128, 0); const __m256i q8sums = __lasx_xvld((const __m256i*)y[i].bsums, 0); const __m128i q8s = lsx_hadd_h(lasx_extracti128(q8sums, 0), lasx_extracti128(q8sums, 1)); - const __m128i prod = lsx_madd_h(lasx_extracti128(mins_and_scales, 1), q8s); + const __m128i prod = lsx_madd_h(mins128, q8s); acc_m = __lsx_vfmadd_s(__lsx_vreplfr2vr_s(dmin), __lsx_vffint_s_w(prod), acc_m); - const __m128i sc128 = lasx_extracti128(mins_and_scales, 0); - const __m256i scales = lasx_insertf128(sc128, sc128); + const __m256i scales = lasx_insertf128(scales128, scales128); __m256i sumi = __lasx_xvldi(0); for (int j = 0; j < QK_K/64; ++j) { - const __m256i scale_l = lasx_shuffle_b(scales, get_scale_shuffle_k4(2*j+0)); - const __m256i scale_h = lasx_shuffle_b(scales, get_scale_shuffle_k4(2*j+1)); + const __m256i scale_l = lasx_xvrepl128vei_h(scales, 2 * j + 0); + const __m256i scale_h = lasx_xvrepl128vei_h(scales, 2 * j + 1); const __m256i q4bits = __lasx_xvld((const __m256i*)q4, 0); q4 += 32; - const __m256i q4l = __lasx_xvand_v(q4bits, m4); - const __m256i q4h = __lasx_xvand_v(__lasx_xvsrli_h(q4bits, 4), m4); + const __m256i q4l = __lasx_xvandi_b(q4bits, 0xf); + const __m256i q4h = __lasx_xvsrli_b(q4bits, 4); const __m256i q8l = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; - __m256i p16l = lasx_maddubs_h(q4l, q8l); + __m256i p16l = lasx_madd_h_b(q4l, q8l); p16l = lasx_madd_h(scale_l, p16l); const __m256i q8h = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; - __m256i p16h = lasx_maddubs_h(q4h, q8h); + __m256i p16h = lasx_madd_h_b(q4h, q8h); p16h = lasx_madd_h(scale_h, p16h); const __m256i sumj = __lasx_xvadd_w(p16l, p16h); @@ -6822,19 +6814,11 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r *s = vec_extract(vsumf0, 0); #elif defined __loongarch_asx - GGML_UNUSED(kmask1); - GGML_UNUSED(kmask2); - GGML_UNUSED(kmask3); - - const __m256i m4 = __lasx_xvreplgr2vr_b(0xF); - const __m128i mzero = __lsx_vldi(0); - const __m256i mone = __lasx_xvreplgr2vr_b(1); __m256 acc = (__m256)__lasx_xvldi(0); + __m128 acc_m = (__m128)__lsx_vldi(0); - float summs = 0.f; - - for (int i = 0; i < nb; ++i) { + for (int i = 0; i < nb; ++i) { const uint8_t * restrict q5 = x[i].qs; const int8_t * restrict q8 = y[i].qs; @@ -6849,49 +6833,40 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r utmp[2] = uaux; utmp[0] &= kmask1; - const __m256i mins_and_scales = lasx_extu8_16(lsx_set_w(utmp[3], utmp[2], utmp[1], utmp[0])); + const __m128i mins_and_scales128 = lsx_set_w(utmp[3], utmp[2], utmp[1], utmp[0]); + const __m128i mins128 = __lsx_vexth_h_b(mins_and_scales128); + const __m128i scales128 = __lsx_vsllwil_h_b(mins_and_scales128, 0); const __m256i q8sums = __lasx_xvld((const __m256i*)y[i].bsums, 0); const __m128i q8s = lsx_hadd_h(lasx_extracti128(q8sums, 0), lasx_extracti128(q8sums, 1)); - const __m128i prod = lsx_madd_h(lasx_extracti128(mins_and_scales, 1), q8s); - const __m128i hsum = lsx_hadd_w(lsx_hadd_w(prod, mzero), mzero); - summs += dmin * __lsx_vpickve2gr_w(hsum, 0); //TODO check + const __m128i prod = lsx_madd_h(mins128, q8s); + acc_m = __lsx_vfmadd_s(__lsx_vreplfr2vr_s(dmin), __lsx_vffint_s_w(prod), acc_m); - const __m128i sc128 = lasx_extracti128(mins_and_scales, 0); - const __m256i scales = lasx_insertf128(sc128, sc128); + const __m256i scales = lasx_insertf128(scales128, scales128); const __m256i hbits = __lasx_xvld((const __m256i*)x[i].qh, 0); - __m256i hmask = mone; __m256i sumi = __lasx_xvldi(0); - int bit = 0; - __m256i xvbit; - for (int j = 0; j < QK_K/64; ++j) { - const __m256i scale_0 = lasx_shuffle_b(scales, get_scale_shuffle_k4(2*j+0)); - const __m256i scale_1 = lasx_shuffle_b(scales, get_scale_shuffle_k4(2*j+1)); + const __m256i scale_0 = lasx_xvrepl128vei_h(scales, 2 * j + 0); + const __m256i scale_1 = lasx_xvrepl128vei_h(scales, 2 * j + 1); const __m256i q5bits = __lasx_xvld((const __m256i*)q5, 0); q5 += 32; - xvbit = __lasx_xvreplgr2vr_h(bit++); - const __m256i q5l_0 = __lasx_xvand_v(q5bits, m4); - const __m256i q5h_0 = __lasx_xvslli_h(__lasx_xvsrl_h(__lasx_xvand_v(hbits, hmask), xvbit), 4); - const __m256i q5_0 = __lasx_xvadd_b(q5l_0, q5h_0); - hmask = __lasx_xvslli_h(hmask, 1); - - xvbit = __lasx_xvreplgr2vr_h(bit++); - const __m256i q5l_1 = __lasx_xvand_v(__lasx_xvsrli_h(q5bits, 4), m4); - const __m256i q5h_1 = __lasx_xvslli_h(__lasx_xvsrl_h(__lasx_xvand_v(hbits, hmask), xvbit), 4); - const __m256i q5_1 = __lasx_xvadd_b(q5l_1, q5h_1); - hmask = __lasx_xvslli_h(hmask, 1); + const __m256i q5l_0 = __lasx_xvandi_b(q5bits, 0xf); + const __m256i q5l_1 = __lasx_xvsrli_b(q5bits, 4); + const __m256i q5h_0 = __lasx_xvnori_b(__lasx_xvseqi_b(lasx_xvandi_b_bit(hbits, 2 * j + 0), 0), 0xef); + const __m256i q5h_1 = __lasx_xvnori_b(__lasx_xvseqi_b(lasx_xvandi_b_bit(hbits, 2 * j + 1), 0), 0xef); + const __m256i q5_0 = __lasx_xvor_v(q5l_0, q5h_0); + const __m256i q5_1 = __lasx_xvor_v(q5l_1, q5h_1); const __m256i q8_0 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; const __m256i q8_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; - __m256i p16_0 = lasx_maddubs_h(q5_0, q8_0); - __m256i p16_1 = lasx_maddubs_h(q5_1, q8_1); + __m256i p16_0 = lasx_madd_h_b(q5_0, q8_0); + __m256i p16_1 = lasx_madd_h_b(q5_1, q8_1); p16_0 = lasx_madd_h(scale_0, p16_0); p16_1 = lasx_madd_h(scale_1, p16_1); @@ -6905,7 +6880,10 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * restrict s, size_t bs, const void * r } - *s = hsum_float_8(acc) + summs; + acc_m = __lsx_vfadd_s(acc_m, (__m128)__lsx_vbsrl_v(acc_m, 8)); + acc_m = __lsx_vfadd_s(acc_m, (__m128)__lsx_vbsrl_v(acc_m, 4)); + + *s = hsum_float_8(acc) + ((v4f32)acc_m)[0]; #elif defined(__VXE__) || defined(__VXE2__) const uint8x16_t v_lm = vec_splat_u8(0x0F); const uint8x16_t v_1m = vec_splat_u8(0x01); @@ -7574,8 +7552,6 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * r #elif defined __loongarch_asx - const __m256i m4 = __lasx_xvreplgr2vr_b(0xF); - const __m256i m2 = __lasx_xvreplgr2vr_b(3); const __m256i m32s = __lasx_xvreplgr2vr_b(32); __m256 acc = (__m256)__lasx_xvldi(0); @@ -7588,58 +7564,42 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * restrict s, size_t bs, const void * r const uint8_t * restrict qh = x[i].qh; const int8_t * restrict q8 = y[i].qs; - const __m128i scales = __lsx_vld((const __m128i*)x[i].scales, 0); + const __m128i scales128 = __lsx_vld((const __m128i*)x[i].scales, 0); + const v16i8 shuffle_mask = {0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15}; + const __m256i scales_shuffled = lasx_ext8_16(__lsx_vshuf_b(scales128, scales128, (__m128i)shuffle_mask)); __m256i sumi = __lasx_xvldi(0); - int is = 0; - for (int j = 0; j < QK_K/128; ++j) { - const __m128i scale_0 = lsx_shuffle_b(scales, get_scale_shuffle(is + 0)); - const __m128i scale_1 = lsx_shuffle_b(scales, get_scale_shuffle(is + 1)); - const __m128i scale_2 = lsx_shuffle_b(scales, get_scale_shuffle(is + 2)); - const __m128i scale_3 = lsx_shuffle_b(scales, get_scale_shuffle(is + 3)); - is += 4; - const __m256i q4bits1 = __lasx_xvld((const __m256i*)q4, 0); q4 += 32; const __m256i q4bits2 = __lasx_xvld((const __m256i*)q4, 0); q4 += 32; const __m256i q4bitsH = __lasx_xvld((const __m256i*)qh, 0); qh += 32; - const __m256i q4h_0 = __lasx_xvslli_h(__lasx_xvand_v(q4bitsH, m2), 4); - const __m256i q4h_1 = __lasx_xvslli_h(__lasx_xvand_v(__lasx_xvsrli_h(q4bitsH, 2), m2), 4); - const __m256i q4h_2 = __lasx_xvslli_h(__lasx_xvand_v(__lasx_xvsrli_h(q4bitsH, 4), m2), 4); - const __m256i q4h_3 = __lasx_xvslli_h(__lasx_xvand_v(__lasx_xvsrli_h(q4bitsH, 6), m2), 4); + const __m256i q4h_0 = __lasx_xvslli_b(__lasx_xvandi_b(q4bitsH, 3), 4); + const __m256i q4h_1 = __lasx_xvslli_b(__lasx_xvandi_b(q4bitsH, 3 << 2), 2); + const __m256i q4h_2 = __lasx_xvandi_b(q4bitsH, 3 << 4); + const __m256i q4h_3 = __lasx_xvsrli_b(__lasx_xvandi_b(q4bitsH, 3 << 6), 2); - const __m256i q4_0 = __lasx_xvor_v(__lasx_xvand_v(q4bits1, m4), q4h_0); - const __m256i q4_1 = __lasx_xvor_v(__lasx_xvand_v(q4bits2, m4), q4h_1); - const __m256i q4_2 = __lasx_xvor_v(__lasx_xvand_v(__lasx_xvsrli_h(q4bits1, 4), m4), q4h_2); - const __m256i q4_3 = __lasx_xvor_v(__lasx_xvand_v(__lasx_xvsrli_h(q4bits2, 4), m4), q4h_3); + const __m256i q4_0 = __lasx_xvor_v(__lasx_xvandi_b(q4bits1, 0xf), q4h_0); + const __m256i q4_1 = __lasx_xvor_v(__lasx_xvandi_b(q4bits2, 0xf), q4h_1); + const __m256i q4_2 = __lasx_xvor_v(__lasx_xvsrli_b(q4bits1, 4), q4h_2); + const __m256i q4_3 = __lasx_xvor_v(__lasx_xvsrli_b(q4bits2, 4), q4h_3); const __m256i q8_0 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; const __m256i q8_1 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; const __m256i q8_2 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; const __m256i q8_3 = __lasx_xvld((const __m256i*)q8, 0); q8 += 32; - __m256i q8s_0 = lasx_maddubs_h(m32s, q8_0); - __m256i q8s_1 = lasx_maddubs_h(m32s, q8_1); - __m256i q8s_2 = lasx_maddubs_h(m32s, q8_2); - __m256i q8s_3 = lasx_maddubs_h(m32s, q8_3); - - __m256i p16_0 = lasx_maddubs_h(q4_0, q8_0); - __m256i p16_1 = lasx_maddubs_h(q4_1, q8_1); - __m256i p16_2 = lasx_maddubs_h(q4_2, q8_2); - __m256i p16_3 = lasx_maddubs_h(q4_3, q8_3); + __m256i p16_0 = lasx_madd_h_b(__lasx_xvsub_b(q4_0, m32s), q8_0); + __m256i p16_1 = lasx_madd_h_b(__lasx_xvsub_b(q4_1, m32s), q8_1); + __m256i p16_2 = lasx_madd_h_b(__lasx_xvsub_b(q4_2, m32s), q8_2); + __m256i p16_3 = lasx_madd_h_b(__lasx_xvsub_b(q4_3, m32s), q8_3); - p16_0 = __lasx_xvsub_h(p16_0, q8s_0); - p16_1 = __lasx_xvsub_h(p16_1, q8s_1); - p16_2 = __lasx_xvsub_h(p16_2, q8s_2); - p16_3 = __lasx_xvsub_h(p16_3, q8s_3); - - p16_0 = lasx_madd_h(lasx_ext8_16(scale_0), p16_0); - p16_1 = lasx_madd_h(lasx_ext8_16(scale_1), p16_1); - p16_2 = lasx_madd_h(lasx_ext8_16(scale_2), p16_2); - p16_3 = lasx_madd_h(lasx_ext8_16(scale_3), p16_3); + p16_0 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 0), p16_0); + p16_1 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 1), p16_1); + p16_2 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 2), p16_2); + p16_3 = lasx_madd_h(lasx_xvrepl128vei_h(scales_shuffled, 4 * j + 3), p16_3); sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_0, p16_1)); sumi = __lasx_xvadd_w(sumi, __lasx_xvadd_w(p16_2, p16_3)); @@ -10137,13 +10097,9 @@ static inline __m256i mul_add_epi8(const __m256i x, const __m256i y) { } #elif defined(__loongarch_asx) static inline __m256i mul_add_epi8(const __m256i x, const __m256i y) { - const __m256i ax = __lasx_xvsigncov_b(x, x); - const __m256i sy = __lasx_xvsigncov_b(x, y); - __m256i tmp1, tmp2, tmp3; - tmp1 = __lasx_xvmulwev_h_bu_b(ax, sy); - tmp2 = __lasx_xvmulwod_h_bu_b(ax, sy); - tmp3 = __lasx_xvadd_h(tmp1, tmp2); - return __lasx_xvsat_h(tmp3, 15); + const __m256i a = __lasx_xvmulwev_h_b(x, y); + const __m256i b = __lasx_xvmulwod_h_b(x, y); + return __lasx_xvadd_h(a, b); } #endif @@ -11214,67 +11170,31 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * restrict s, size_t bs, const void * #elif defined(__loongarch_asx) const __m128i values128 = __lsx_vld((const __m128i*)kvalues_iq4nl, 0); - const __m128i m4b = __lsx_vreplgr2vr_b(0x0f); __m256 accum = (__m256)__lasx_xvldi(0); - __m256i tmp1; - __m128i tmp0, tmp2, tmp3, tmp4, mask_8f, mask; - mask_8f = __lsx_vreplgr2vr_b(0x8f); for (int ibl = 0; ibl < nb; ++ibl) { const uint8_t * qs = x[ibl].qs; const int8_t * q8 = y[ibl].qs; uint16_t sh = x[ibl].scales_h; __m256i sumi1 = __lasx_xvldi(0); __m256i sumi2 = __lasx_xvldi(0); - __m128i zero = __lsx_vldi(0); for (int ib = 0; ib < QK_K/32; ib += 2) { - const __m128i q4bits_1 = __lsx_vld((const __m128i*)qs, 0); qs += 16; - const __m128i q4bits_2 = __lsx_vld((const __m128i*)qs, 0); qs += 16; + const __m128i q4bits_1 = __lsx_vld((const __m128i*)qs, 0); qs += 16; + const __m128i q4bits_2 = __lsx_vld((const __m128i*)qs, 0); qs += 16; const __m256i q8b_1 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32; const __m256i q8b_2 = __lasx_xvld((const __m256i *)q8, 0); q8 += 32; - tmp2 = __lsx_vand_v(__lsx_vand_v(__lsx_vsrli_h(q4bits_1, 4), m4b), mask_8f); - tmp0 = __lsx_vori_b(tmp2, 0x10); - mask = __lsx_vsle_b(zero, tmp2); - tmp3 = __lsx_vand_v(tmp0, mask); - tmp3 = __lsx_vshuf_b(values128, zero, tmp3); - - tmp2 = __lsx_vand_v(__lsx_vand_v(q4bits_1, m4b), mask_8f); - tmp0 = __lsx_vori_b(tmp2, 0x10); - mask = __lsx_vsle_b(zero, tmp2); - tmp4 = __lsx_vand_v(tmp0, mask); - tmp4 = __lsx_vshuf_b(values128, zero, tmp4); - - const __m256i q4b_1 = lasx_insertf128(tmp3, tmp4); - - tmp2 = __lsx_vand_v(__lsx_vand_v(__lsx_vsrli_h(q4bits_2, 4), m4b), mask_8f); - tmp0 = __lsx_vori_b(tmp2, 0x10); - mask = __lsx_vsle_b(zero, tmp2); - tmp3 = __lsx_vand_v(tmp0, mask); - tmp3 = __lsx_vshuf_b(values128, zero, tmp3); - - tmp2 = __lsx_vand_v(__lsx_vand_v(q4bits_2, m4b), mask_8f); - tmp0 = __lsx_vori_b(tmp2, 0x10); - mask = __lsx_vsle_b(zero, tmp2); - tmp4 = __lsx_vand_v(tmp0, mask); - tmp4 = __lsx_vshuf_b(values128, zero, tmp4); - - const __m256i q4b_2 = lasx_insertf128(tmp3, tmp4); - + const __m256i q4b_1 = lasx_insertf128(__lsx_vshuf_b(values128, values128, __lsx_vsrli_b(q4bits_1, 4)), + __lsx_vshuf_b(values128, values128, __lsx_vandi_b(q4bits_1, 0xf))); + const __m256i q4b_2 = lasx_insertf128(__lsx_vshuf_b(values128, values128, __lsx_vsrli_b(q4bits_2, 4)), + __lsx_vshuf_b(values128, values128, __lsx_vandi_b(q4bits_2, 0xf))); const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1); const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2); const int16_t ls1 = ((x[ibl].scales_l[ib/2] & 0xf) | ((sh << 4) & 0x30)) - 32; const int16_t ls2 = ((x[ibl].scales_l[ib/2] >> 4) | ((sh << 2) & 0x30)) - 32; sh >>= 4; - __m256i tmp5, tmp6; - tmp1 = __lasx_xvreplgr2vr_h(ls1); - tmp5 = __lasx_xvmulwev_w_h(p16_1, tmp1); - tmp6 = __lasx_xvmulwod_w_h(p16_1, tmp1); - const __m256i p_1 = __lasx_xvadd_w(tmp5, tmp6); - tmp1 = __lasx_xvreplgr2vr_h(ls2); - tmp5 = __lasx_xvmulwev_w_h(p16_2, tmp1); - tmp6 = __lasx_xvmulwod_w_h(p16_2, tmp1); - const __m256i p_2 = __lasx_xvadd_w(tmp5, tmp6); + const __m256i p_1 = lasx_madd_h(p16_1, __lasx_xvreplgr2vr_h(ls1)); + const __m256i p_2 = lasx_madd_h(p16_2, __lasx_xvreplgr2vr_h(ls2)); sumi1 = __lasx_xvadd_w(p_1, sumi1); sumi2 = __lasx_xvadd_w(p_2, sumi2); } From 3a42a05c9c01b2a3b74966c6bb80d1f75cde86fe Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Sat, 22 Feb 2025 22:07:36 +0800 Subject: [PATCH 56/56] ggml: cmake remove fork when determining s390x machine type thank you @ericcurtin Signed-off-by: Aaron Teo --- ggml/src/ggml-cpu/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index 6d6e3049614b7..aa5ad5d8d9050 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -312,7 +312,8 @@ function(ggml_add_cpu_backend_variant_impl tag_name) endif() elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "s390x") message(STATUS "s390x detected") - execute_process(COMMAND bash -c "grep -Pom 1 'machine = \\K([0-9]+)' /proc/cpuinfo" OUTPUT_VARIABLE S390X_M) + file(READ "/proc/cpuinfo" CPUINFO_CONTENTS) + string(REGEX REPLACE "machine[ \t\r\n]*=[ \t\r\n]*([0-9]+)" "\\1" S390X_M ${CPUINFO_CONTENTS}) # TODO: Separation to determine activation of VX/VXE/VXE2 if (${S390X_M} MATCHES "8561|8562")