Skip to content

Commit e1b7123

Browse files
martin-frbgmartin-frbg
authored
Merge pull request #2867 from Qiyu8/usimd-floatdot
Optimize the performance of dot by using universal intrinsics in X86/ARM
2 parents 599777e + f32d34a commit e1b7123

File tree

13 files changed

+193
-43
lines changed

13 files changed

+193
-43
lines changed

‎Makefile.x86_64

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,11 @@ endif
88
endif
99
endif
1010

11+
ifdef HAVE_SSE3
12+
CCOMMON_OPT += -msse3
13+
FCOMMON_OPT += -msse3
14+
endif
15+
1116
ifeq ($(CORE), SKYLAKEX)
1217
ifndef DYNAMIC_ARCH
1318
ifndef NO_AVX512

‎cmake/system.cmake

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,9 @@ if (DEFINED TARGET)
7070
set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -mavx2")
7171
endif()
7272
endif()
73+
if (DEFINED HAVE_SSE3)
74+
set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -msse3")
75+
endif()
7376
endif()
7477

7578
if (DEFINED TARGET)

‎kernel/Makefile

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@ endif
55
TOPDIR = ..
66
include $(TOPDIR)/Makefile.system
77

8+
ifdef HAVE_SSE3
9+
CFLAGS += -msse3
10+
endif
811

912
ifeq ($(C_COMPILER), GCC)
1013
GCCVERSIONGTEQ9 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \>= 9)

‎kernel/arm64/KERNEL.ARMV8

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ CNRM2KERNEL = znrm2.S
9797
ZNRM2KERNEL = znrm2.S
9898

9999
DDOTKERNEL = dot.S
100-
SDOTKERNEL = dot.S
100+
SDOTKERNEL = ../generic/dot.c
101101
CDOTKERNEL = zdot.S
102102
ZDOTKERNEL = zdot.S
103103
DSDOTKERNEL = dot.S

‎kernel/arm64/KERNEL.CORTEXA53

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ CNRM2KERNEL = znrm2.S
9797
ZNRM2KERNEL = znrm2.S
9898

9999
DDOTKERNEL = dot.S
100-
SDOTKERNEL = dot.S
100+
SDOTKERNEL = ../generic/dot.c
101101
CDOTKERNEL = zdot.S
102102
ZDOTKERNEL = zdot.S
103103
DSDOTKERNEL = dot.S

‎kernel/arm64/KERNEL.CORTEXA57

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ DCOPYKERNEL = copy.S
7070
CCOPYKERNEL = copy.S
7171
ZCOPYKERNEL = copy.S
7272

73-
SDOTKERNEL = dot.S
73+
SDOTKERNEL = ../generic/dot.c
7474
DDOTKERNEL = dot.S
7575
CDOTKERNEL = zdot.S
7676
ZDOTKERNEL = zdot.S

‎kernel/generic/dot.c

Lines changed: 43 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
2727

2828

2929
#include "common.h"
30-
30+
#include "../simd/intrin.h"
3131
#if defined(DSDOT)
3232
double CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y)
3333
#else
@@ -47,27 +47,59 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y)
4747

4848
if ( (inc_x == 1) && (inc_y == 1) )
4949
{
50-
51-
int n1 = n & -4;
52-
53-
while(i < n1)
50+
int n1 = n & -4;
51+
#if V_SIMD && !defined(DSDOT)
52+
const int vstep = v_nlanes_f32;
53+
const int unrollx4 = n & (-vstep * 4);
54+
const int unrollx = n & -vstep;
55+
v_f32 vsum0 = v_zero_f32();
56+
v_f32 vsum1 = v_zero_f32();
57+
v_f32 vsum2 = v_zero_f32();
58+
v_f32 vsum3 = v_zero_f32();
59+
while(i < unrollx4)
60+
{
61+
vsum0 = v_muladd_f32(
62+
v_loadu_f32(x + i), v_loadu_f32(y + i), vsum0
63+
);
64+
vsum1 = v_muladd_f32(
65+
v_loadu_f32(x + i + vstep), v_loadu_f32(y + i + vstep), vsum1
66+
);
67+
vsum2 = v_muladd_f32(
68+
v_loadu_f32(x + i + vstep*2), v_loadu_f32(y + i + vstep*2), vsum2
69+
);
70+
vsum3 = v_muladd_f32(
71+
v_loadu_f32(x + i + vstep*3), v_loadu_f32(y + i + vstep*3), vsum3
72+
);
73+
i += vstep*4;
74+
}
75+
vsum0 = v_add_f32(
76+
v_add_f32(vsum0, vsum1), v_add_f32(vsum2 , vsum3)
77+
);
78+
while(i < unrollx)
79+
{
80+
vsum0 = v_muladd_f32(
81+
v_loadu_f32(x + i), v_loadu_f32(y + i), vsum0
82+
);
83+
i += vstep;
84+
}
85+
dot = v_sum_f32(vsum0);
86+
#elif defined(DSDOT)
87+
for (; i < n1; i += 4)
5488
{
55-
56-
#if defined(DSDOT)
5789
dot += (double) y[i] * (double) x[i]
5890
+ (double) y[i+1] * (double) x[i+1]
5991
+ (double) y[i+2] * (double) x[i+2]
6092
+ (double) y[i+3] * (double) x[i+3] ;
93+
}
6194
#else
95+
for (; i < n1; i += 4)
96+
{
6297
dot += y[i] * x[i]
6398
+ y[i+1] * x[i+1]
6499
+ y[i+2] * x[i+2]
65100
+ y[i+3] * x[i+3] ;
66-
#endif
67-
i+=4 ;
68-
69101
}
70-
102+
#endif
71103
while(i < n)
72104
{
73105

‎kernel/simd/intrin.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,11 @@ extern "C" {
5151
#include <immintrin.h>
5252
#endif
5353

54+
/** NEON **/
55+
#ifdef HAVE_NEON
56+
#include <arm_neon.h>
57+
#endif
58+
5459
// distribute
5560
#if defined(HAVE_AVX512VL) || defined(HAVE_AVX512BF16)
5661
#include "intrin_avx512.h"
@@ -60,6 +65,10 @@ extern "C" {
6065
#include "intrin_sse.h"
6166
#endif
6267

68+
#ifdef HAVE_NEON
69+
#include "intrin_neon.h"
70+
#endif
71+
6372
#ifndef V_SIMD
6473
#define V_SIMD 0
6574
#define V_SIMD_F64 0

‎kernel/simd/intrin_avx.h

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
#define V_SIMD 256
22
#define V_SIMD_F64 1
3-
/*
4-
Data Type
5-
*/
3+
/***************************
4+
* Data Type
5+
***************************/
66
typedef __m256 v_f32;
77
#define v_nlanes_f32 8
8-
/*
9-
arithmetic
10-
*/
8+
/***************************
9+
* Arithmetic
10+
***************************/
1111
#define v_add_f32 _mm256_add_ps
1212
#define v_mul_f32 _mm256_mul_ps
1313

@@ -20,10 +20,22 @@ arithmetic
2020
{ return v_add_f32(v_mul_f32(a, b), c); }
2121
#endif // !HAVE_FMA3
2222

23-
/*
24-
memory
25-
*/
23+
// Horizontal add: Calculates the sum of all vector elements.
24+
BLAS_FINLINE float v_sum_f32(__m256 a)
25+
{
26+
__m256 sum_halves = _mm256_hadd_ps(a, a);
27+
sum_halves = _mm256_hadd_ps(sum_halves, sum_halves);
28+
__m128 lo = _mm256_castps256_ps128(sum_halves);
29+
__m128 hi = _mm256_extractf128_ps(sum_halves, 1);
30+
__m128 sum = _mm_add_ps(lo, hi);
31+
return _mm_cvtss_f32(sum);
32+
}
33+
34+
/***************************
35+
* memory
36+
***************************/
2637
// unaligned load
2738
#define v_loadu_f32 _mm256_loadu_ps
2839
#define v_storeu_f32 _mm256_storeu_ps
29-
#define v_setall_f32(VAL) _mm256_set1_ps(VAL)
40+
#define v_setall_f32(VAL) _mm256_set1_ps(VAL)
41+
#define v_zero_f32 _mm256_setzero_ps

‎kernel/simd/intrin_avx512.h

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,35 @@
11
#define V_SIMD 512
22
#define V_SIMD_F64 1
3-
/*
4-
Data Type
5-
*/
3+
/***************************
4+
* Data Type
5+
***************************/
66
typedef __m512 v_f32;
77
#define v_nlanes_f32 16
8-
/*
9-
arithmetic
10-
*/
8+
/***************************
9+
* Arithmetic
10+
***************************/
1111
#define v_add_f32 _mm512_add_ps
1212
#define v_mul_f32 _mm512_mul_ps
1313
// multiply and add, a*b + c
1414
#define v_muladd_f32 _mm512_fmadd_ps
15-
/*
16-
memory
17-
*/
15+
16+
BLAS_FINLINE float v_sum_f32(v_f32 a)
17+
{
18+
__m512 h64 = _mm512_shuffle_f32x4(a, a, _MM_SHUFFLE(3, 2, 3, 2));
19+
__m512 sum32 = _mm512_add_ps(a, h64);
20+
__m512 h32 = _mm512_shuffle_f32x4(sum32, sum32, _MM_SHUFFLE(1, 0, 3, 2));
21+
__m512 sum16 = _mm512_add_ps(sum32, h32);
22+
__m512 h16 = _mm512_permute_ps(sum16, _MM_SHUFFLE(1, 0, 3, 2));
23+
__m512 sum8 = _mm512_add_ps(sum16, h16);
24+
__m512 h4 = _mm512_permute_ps(sum8, _MM_SHUFFLE(2, 3, 0, 1));
25+
__m512 sum4 = _mm512_add_ps(sum8, h4);
26+
return _mm_cvtss_f32(_mm512_castps512_ps128(sum4));
27+
}
28+
/***************************
29+
* memory
30+
***************************/
1831
// unaligned load
1932
#define v_loadu_f32(PTR) _mm512_loadu_ps((const __m512*)(PTR))
2033
#define v_storeu_f32 _mm512_storeu_ps
2134
#define v_setall_f32(VAL) _mm512_set1_ps(VAL)
35+
#define v_zero_f32 _mm512_setzero_ps

0 commit comments

Comments
 (0)