Skip to content

Commit 17b032f

Browse files
taronaeotaronaeo
committed
ggml: refactor fp16<->fp32 simd to ggml-cpu
Signed-off-by: Aaron Teo <[email protected]>
1 parent 8a5e011 commit 17b032f

File tree

10 files changed

+255
-182
lines changed

10 files changed

+255
-182
lines changed

‎ggml/src/ggml-cpu/arch/arm/quants.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
#include "../../quants.h"
88
#include "../../ggml-cpu-impl.h"
9+
#include "../../simd-mappings.h"
910

1011
#include <math.h>
1112
#include <string.h>

‎ggml/src/ggml-cpu/arch/arm/repack.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#include "ggml-impl.h"
77
#include "ggml-cpu.h"
88
#include "ggml-cpu-impl.h"
9+
#include "simd-mappings.h"
910
#include "traits.h"
1011

1112
#include <cmath>

‎ggml/src/ggml-cpu/common.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include "traits.h"
55
#include "ggml-cpu-impl.h"
66
#include "ggml-impl.h"
7+
#include "simd-mappings.h"
78

89
#ifdef __cplusplus
910

‎ggml/src/ggml-cpu/llamafile/sgemm.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
#include "ggml-impl.h"
5353
#include "ggml-cpu-impl.h"
5454
#include "ggml-quants.h"
55+
#include "simd-mappings.h"
5556

5657
#include <array>
5758
#include <type_traits>

‎ggml/src/ggml-cpu/quants.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
#include "ggml-common.h"
33

44
#include "ggml-cpu-impl.h"
5+
#include "simd-mappings.h"
56
#include "ggml-quants.h"
67
#include "quants.h"
78

‎ggml/src/ggml-cpu/repack.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
#include "ggml-impl.h"
77
#include "ggml-cpu.h"
88
#include "ggml-cpu-impl.h"
9+
#include "simd-mappings.h"
910
#include "traits.h"
1011

1112
#include "arch-fallback.h"

‎ggml/src/ggml-cpu/simd-mappings.h‎

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,196 @@
22

33
#include "ggml-cpu-impl.h"
44

5+
#ifdef __ARM_FEATURE_SVE
6+
#include <arm_sve.h>
7+
#endif // __ARM_FEATURE_SVE
8+
9+
#if defined(__ARM_NEON) && !defined(__CUDACC__) && !defined(__MUSACC__)
10+
// if YCM cannot find <arm_neon.h>, make a symbolic link to it, for example:
11+
//
12+
// $ ln -sfn /Library/Developer/CommandLineTools/usr/lib/clang/13.1.6/include/arm_neon.h ./src/
13+
//
14+
#include <arm_neon.h>
15+
#endif
16+
17+
#if defined(__F16C__)
18+
#include <immintrin.h>
19+
#endif
20+
521
//
622
// simd mappings
723
//
824

25+
// FP16 to FP32 conversion
26+
27+
// 16-bit float
28+
// on Arm, we use __fp16
29+
// on x86, we use uint16_t
30+
//
31+
// for old CUDA compilers (<= 11), we use uint16_t: ref https://github.com/ggml-org/llama.cpp/pull/10616
32+
// for MUSA compilers , we use uint16_t: ref https://github.com/ggml-org/llama.cpp/pull/11843
33+
//
34+
#if defined(__ARM_NEON) && !(defined(__CUDACC__) && __CUDACC_VER_MAJOR__ <= 11) && !defined(__MUSACC__)
35+
#ifdef GGML_FP16_TO_FP32
36+
#undef GGML_FP16_TO_FP32
37+
#endif
38+
39+
#ifdef GGML_FP32_TO_FP16
40+
#undef GGML_FP32_TO_FP16
41+
#endif
42+
43+
#ifdef GGML_COMPUTE_FP16_TO_FP32
44+
#undef GGML_COMPUTE_FP16_TO_FP32
45+
#endif
46+
47+
#ifdef GGML_COMPUTE_FP32_TO_FP16
48+
#undef GGML_COMPUTE_FP32_TO_FP16
49+
#endif
50+
51+
#define GGML_COMPUTE_FP16_TO_FP32(x) neon_compute_fp16_to_fp32(x)
52+
#define GGML_COMPUTE_FP32_TO_FP16(x) neon_compute_fp32_to_fp16(x)
53+
54+
#define GGML_FP16_TO_FP32(x) neon_compute_fp16_to_fp32(x)
55+
56+
static inline float neon_compute_fp16_to_fp32(ggml_fp16_t h) {
57+
__fp16 tmp;
58+
memcpy(&tmp, &h, sizeof(ggml_fp16_t));
59+
return (float)tmp;
60+
}
61+
62+
static inline ggml_fp16_t neon_compute_fp32_to_fp16(float f) {
63+
ggml_fp16_t res;
64+
__fp16 tmp = f;
65+
memcpy(&res, &tmp, sizeof(ggml_fp16_t));
66+
return res;
67+
}
68+
#elif defined(__F16C__)
69+
#ifdef GGML_COMPUTE_FP16_TO_FP32
70+
#undef GGML_COMPUTE_FP16_TO_FP32
71+
#endif
72+
73+
#ifdef GGML_COMPUTE_FP32_TO_FP16
74+
#undef GGML_COMPUTE_FP32_TO_FP16
75+
#endif
76+
77+
#ifdef _MSC_VER
78+
#define GGML_COMPUTE_FP16_TO_FP32(x) _mm_cvtss_f32(_mm_cvtph_ps(_mm_cvtsi32_si128(x)))
79+
#define GGML_COMPUTE_FP32_TO_FP16(x) _mm_extract_epi16(_mm_cvtps_ph(_mm_set_ss(x), 0), 0)
80+
#else
81+
#define GGML_COMPUTE_FP16_TO_FP32(x) _cvtsh_ss(x)
82+
#define GGML_COMPUTE_FP32_TO_FP16(x) _cvtss_sh(x, 0)
83+
#endif
84+
#elif defined(__POWER9_VECTOR__)
85+
#ifdef GGML_FP16_TO_FP32
86+
#undef GGML_FP16_TO_FP32
87+
#endif
88+
89+
#ifdef GGML_FP32_TO_FP16
90+
#undef GGML_FP32_TO_FP16
91+
#endif
92+
93+
#ifdef GGML_COMPUTE_FP16_TO_FP32
94+
#undef GGML_COMPUTE_FP16_TO_FP32
95+
#endif
96+
97+
#ifdef GGML_COMPUTE_FP32_TO_FP16
98+
#undef GGML_COMPUTE_FP32_TO_FP16
99+
#endif
100+
101+
#define GGML_COMPUTE_FP16_TO_FP32(x) power_compute_fp16_to_fp32(x)
102+
#define GGML_COMPUTE_FP32_TO_FP16(x) power_compute_fp32_to_fp16(x)
103+
/* the inline asm below is about 12% faster than the lookup method */
104+
#define GGML_FP16_TO_FP32(x) GGML_COMPUTE_FP16_TO_FP32(x)
105+
#define GGML_FP32_TO_FP16(x) GGML_COMPUTE_FP32_TO_FP16(x)
106+
107+
static inline float power_compute_fp16_to_fp32(ggml_fp16_t h) {
108+
float f;
109+
double d;
110+
__asm__(
111+
"mtfprd %0,%2\n"
112+
"xscvhpdp %0,%0\n"
113+
"frsp %1,%0\n" :
114+
/* temp */ "=d"(d),
115+
/* out */ "=f"(f):
116+
/* in */ "r"(h));
117+
return f;
118+
}
119+
120+
static inline ggml_fp16_t power_compute_fp32_to_fp16(float f) {
121+
double d;
122+
ggml_fp16_t r;
123+
__asm__( /* xscvdphp can work on double or single precision */
124+
"xscvdphp %0,%2\n"
125+
"mffprd %1,%0\n" :
126+
/* temp */ "=d"(d),
127+
/* out */ "=r"(r):
128+
/* in */ "f"(f));
129+
return r;
130+
}
131+
132+
#elif defined(__riscv) && defined(__riscv_zfhmin)
133+
#ifdef GGML_FP16_TO_FP32
134+
#undef GGML_FP16_TO_FP32
135+
#endif
136+
137+
#ifdef GGML_FP32_TO_FP16
138+
#undef GGML_FP32_TO_FP16
139+
#endif
140+
141+
#ifdef GGML_COMPUTE_FP16_TO_FP32
142+
#undef GGML_COMPUTE_FP16_TO_FP32
143+
#endif
144+
145+
#ifdef GGML_COMPUTE_FP32_TO_FP16
146+
#undef GGML_COMPUTE_FP32_TO_FP16
147+
#endif
148+
149+
static inline float riscv_compute_fp16_to_fp32(ggml_fp16_t h) {
150+
float f;
151+
__asm__(
152+
"fmv.h.x %[f], %[h]\n\t"
153+
"fcvt.s.h %[f], %[f]"
154+
: [f] "=&f" (f)
155+
: [h] "r" (h)
156+
);
157+
return f;
158+
}
159+
160+
static inline ggml_fp16_t riscv_compute_fp32_to_fp16(float f) {
161+
ggml_fp16_t res;
162+
__asm__(
163+
"fcvt.h.s %[f], %[f]\n\t"
164+
"fmv.x.h %[h], %[f]"
165+
: [h] "=&r" (res)
166+
: [f] "f" (f)
167+
);
168+
return res;
169+
}
170+
171+
#define GGML_COMPUTE_FP16_TO_FP32(x) riscv_compute_fp16_to_fp32(x)
172+
#define GGML_COMPUTE_FP32_TO_FP16(x) riscv_compute_fp32_to_fp16(x)
173+
#define GGML_FP16_TO_FP32(x) GGML_COMPUTE_FP16_TO_FP32(x)
174+
#define GGML_FP32_TO_FP16(x) GGML_COMPUTE_FP32_TO_FP16(x)
175+
#endif
176+
177+
// On ARM NEON, it's quicker to directly convert x -> x instead of calling into ggml_lookup_fp16_to_fp32,
178+
// so we define GGML_FP16_TO_FP32 and GGML_FP32_TO_FP16 elsewhere for NEON.
179+
// This is also true for POWER9.
180+
#if !defined(GGML_FP16_TO_FP32)
181+
inline static float ggml_lookup_fp16_to_fp32(ggml_fp16_t f) {
182+
uint16_t s;
183+
memcpy(&s, &f, sizeof(uint16_t));
184+
return ggml_table_f32_f16[s];
185+
}
186+
187+
#define GGML_FP16_TO_FP32(x) ggml_lookup_fp16_to_fp32(x)
188+
#endif
189+
190+
#if !defined(GGML_FP32_TO_FP16)
191+
#define GGML_FP32_TO_FP16(x) GGML_COMPUTE_FP32_TO_FP16(x)
192+
#endif
193+
194+
9195
// we define a common set of C macros which map to specific intrinsics based on the current architecture
10196
// we then implement the fundamental computation operations below using only these macros
11197
// adding support for new architectures requires to define the corresponding SIMD macros

0 commit comments

Comments
 (0)