Skip to content

Commit 989c0d2

Browse files
moorabbitRKSimon
moorabbit
and
RKSimon
authored
[Clang][X86] Replace unnecessary vfmadd* builtins with element_wise_fma (#152545)
The following intrinsics were replaced by `__builtin_elementwise_fma`: - `__builtin_ia32_vfmaddps(256)` - `__builtin_ia32_vfmaddpd(256)` - `__builtin_ia32_vfmaddph(256)` - `__builtin_ia32_vfmaddbf16(128 | 256 | 512)` All the aforementioned `__builtin_ia32_vfmadd*` intrinsics are equivalent to a `__builtin_elementwise_fma`, so keeping them is an unnecessary indirection. Fixes [#152461](llvm/llvm-project#152461) --------- Co-authored-by: Simon Pilgrim <[email protected]>
1 parent 6894abe commit 989c0d2

File tree

9 files changed

+323
-371
lines changed

9 files changed

+323
-371
lines changed

‎clang/include/clang/Basic/BuiltinsX86.td‎

Lines changed: 2 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -884,11 +884,6 @@ let Features = "sha", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in
884884
def sha256msg2 : X86Builtin<"_Vector<4, int>(_Vector<4, int>, _Vector<4, int>)">;
885885
}
886886

887-
let Features = "fma|fma4", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
888-
def vfmaddps : X86Builtin<"_Vector<4, float>(_Vector<4, float>, _Vector<4, float>, _Vector<4, float>)">;
889-
def vfmaddpd : X86Builtin<"_Vector<2, double>(_Vector<2, double>, _Vector<2, double>, _Vector<2, double>)">;
890-
}
891-
892887
let Features = "fma", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
893888
def vfmaddss3 : X86Builtin<"_Vector<4, float>(_Vector<4, float>, _Vector<4, float>, _Vector<4, float>)">;
894889
def vfmaddsd3 : X86Builtin<"_Vector<2, double>(_Vector<2, double>, _Vector<2, double>, _Vector<2, double>)">;
@@ -904,9 +899,8 @@ let Features = "fma|fma4", Attributes = [NoThrow, Const, RequiredVectorWidth<128
904899
def vfmaddsubpd : X86Builtin<"_Vector<2, double>(_Vector<2, double>, _Vector<2, double>, _Vector<2, double>)">;
905900
}
906901

907-
let Features = "fma|fma4", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
908-
def vfmaddps256 : X86Builtin<"_Vector<8, float>(_Vector<8, float>, _Vector<8, float>, _Vector<8, float>)">;
909-
def vfmaddpd256 : X86Builtin<"_Vector<4, double>(_Vector<4, double>, _Vector<4, double>, _Vector<4, double>)">;
902+
let Features = "fma|fma4",
903+
Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
910904
def vfmaddsubps256 : X86Builtin<"_Vector<8, float>(_Vector<8, float>, _Vector<8, float>, _Vector<8, float>)">;
911905
def vfmaddsubpd256 : X86Builtin<"_Vector<4, double>(_Vector<4, double>, _Vector<4, double>, _Vector<4, double>)">;
912906
}
@@ -4149,14 +4143,6 @@ let Features = "avx512fp16,evex512", Attributes = [NoThrow, Const, RequiredVecto
41494143
def vcvtps2phx512_mask : X86Builtin<"_Vector<16, _Float16>(_Vector<16, float>, _Vector<16, _Float16>, unsigned short, _Constant int)">;
41504144
}
41514145

4152-
let Features = "avx512fp16,avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
4153-
def vfmaddph : X86Builtin<"_Vector<8, _Float16>(_Vector<8, _Float16>, _Vector<8, _Float16>, _Vector<8, _Float16>)">;
4154-
}
4155-
4156-
let Features = "avx512fp16,avx512vl", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
4157-
def vfmaddph256 : X86Builtin<"_Vector<16, _Float16>(_Vector<16, _Float16>, _Vector<16, _Float16>, _Vector<16, _Float16>)">;
4158-
}
4159-
41604146
let Features = "avx512fp16,evex512", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
41614147
def vfmaddph512_mask : X86Builtin<"_Vector<32, _Float16>(_Vector<32, _Float16>, _Vector<32, _Float16>, _Vector<32, _Float16>, unsigned int, _Constant int)">;
41624148
def vfmaddph512_mask3 : X86Builtin<"_Vector<32, _Float16>(_Vector<32, _Float16>, _Vector<32, _Float16>, _Vector<32, _Float16>, unsigned int, _Constant int)">;
@@ -5382,13 +5368,4 @@ let Features = "avx10.2-256", Attributes = [NoThrow, Const, RequiredVectorWidth<
53825368

53835369
let Features = "avx10.2-512", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
53845370
def vsqrtbf16512 : X86Builtin<"_Vector<32, __bf16>(_Vector<32, __bf16>)">;
5385-
def vfmaddbf16512 : X86Builtin<"_Vector<32, __bf16>(_Vector<32, __bf16>, _Vector<32, __bf16>, _Vector<32, __bf16>)">;
5386-
}
5387-
5388-
let Features = "avx10.2-256", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
5389-
def vfmaddbf16256 : X86Builtin<"_Vector<16, __bf16>(_Vector<16, __bf16>, _Vector<16, __bf16>, _Vector<16, __bf16>)">;
5390-
}
5391-
5392-
let Features = "avx10.2-256", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
5393-
def vfmaddbf16128 : X86Builtin<"_Vector<8, __bf16>(_Vector<8, __bf16>, _Vector<8, __bf16>, _Vector<8, __bf16>)">;
53945371
}

‎clang/lib/CodeGen/TargetBuiltins/X86.cpp‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1051,18 +1051,9 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
10511051
case X86::BI__builtin_ia32_vfmsubsd3_mask3:
10521052
return EmitScalarFMAExpr(*this, E, Ops, Ops[2], /*ZeroMask*/ false, 2,
10531053
/*NegAcc*/ true);
1054-
case X86::BI__builtin_ia32_vfmaddph:
1055-
case X86::BI__builtin_ia32_vfmaddps:
1056-
case X86::BI__builtin_ia32_vfmaddpd:
1057-
case X86::BI__builtin_ia32_vfmaddph256:
1058-
case X86::BI__builtin_ia32_vfmaddps256:
1059-
case X86::BI__builtin_ia32_vfmaddpd256:
10601054
case X86::BI__builtin_ia32_vfmaddph512_mask:
10611055
case X86::BI__builtin_ia32_vfmaddph512_maskz:
10621056
case X86::BI__builtin_ia32_vfmaddph512_mask3:
1063-
case X86::BI__builtin_ia32_vfmaddbf16128:
1064-
case X86::BI__builtin_ia32_vfmaddbf16256:
1065-
case X86::BI__builtin_ia32_vfmaddbf16512:
10661057
case X86::BI__builtin_ia32_vfmaddps512_mask:
10671058
case X86::BI__builtin_ia32_vfmaddps512_maskz:
10681059
case X86::BI__builtin_ia32_vfmaddps512_mask3:

‎clang/lib/Headers/avx10_2_512bf16intrin.h‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -441,8 +441,8 @@ _mm512_maskz_sqrt_pbh(__mmask32 __U, __m512bh __A) {
441441

442442
static __inline__ __m512bh __DEFAULT_FN_ATTRS512
443443
_mm512_fmadd_pbh(__m512bh __A, __m512bh __B, __m512bh __C) {
444-
return (__m512bh)__builtin_ia32_vfmaddbf16512((__v32bf)__A, (__v32bf)__B,
445-
(__v32bf)__C);
444+
return (__m512bh)__builtin_elementwise_fma((__v32bf)__A, (__v32bf)__B,
445+
(__v32bf)__C);
446446
}
447447

448448
static __inline__ __m512bh __DEFAULT_FN_ATTRS512
@@ -469,8 +469,8 @@ static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_maskz_fmadd_pbh(
469469

470470
static __inline__ __m512bh __DEFAULT_FN_ATTRS512
471471
_mm512_fmsub_pbh(__m512bh __A, __m512bh __B, __m512bh __C) {
472-
return (__m512bh)__builtin_ia32_vfmaddbf16512((__v32bf)__A, (__v32bf)__B,
473-
-(__v32bf)__C);
472+
return (__m512bh)__builtin_elementwise_fma((__v32bf)__A, (__v32bf)__B,
473+
-(__v32bf)__C);
474474
}
475475

476476
static __inline__ __m512bh __DEFAULT_FN_ATTRS512
@@ -497,8 +497,8 @@ static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_maskz_fmsub_pbh(
497497

498498
static __inline__ __m512bh __DEFAULT_FN_ATTRS512
499499
_mm512_fnmadd_pbh(__m512bh __A, __m512bh __B, __m512bh __C) {
500-
return (__m512bh)__builtin_ia32_vfmaddbf16512((__v32bf)__A, -(__v32bf)__B,
501-
(__v32bf)__C);
500+
return (__m512bh)__builtin_elementwise_fma((__v32bf)__A, -(__v32bf)__B,
501+
(__v32bf)__C);
502502
}
503503

504504
static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_mask_fnmadd_pbh(
@@ -527,8 +527,8 @@ static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_maskz_fnmadd_pbh(
527527

528528
static __inline__ __m512bh __DEFAULT_FN_ATTRS512
529529
_mm512_fnmsub_pbh(__m512bh __A, __m512bh __B, __m512bh __C) {
530-
return (__m512bh)__builtin_ia32_vfmaddbf16512((__v32bf)__A, -(__v32bf)__B,
531-
-(__v32bf)__C);
530+
return (__m512bh)__builtin_elementwise_fma((__v32bf)__A, -(__v32bf)__B,
531+
-(__v32bf)__C);
532532
}
533533

534534
static __inline__ __m512bh __DEFAULT_FN_ATTRS512 _mm512_mask_fnmsub_pbh(

‎clang/lib/Headers/avx10_2bf16intrin.h‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -852,8 +852,8 @@ _mm_maskz_sqrt_pbh(__mmask8 __U, __m128bh __A) {
852852

853853
static __inline__ __m256bh __DEFAULT_FN_ATTRS256
854854
_mm256_fmadd_pbh(__m256bh __A, __m256bh __B, __m256bh __C) {
855-
return (__m256bh)__builtin_ia32_vfmaddbf16256((__v16bf)__A, (__v16bf)__B,
856-
(__v16bf)__C);
855+
return (__m256bh)__builtin_elementwise_fma((__v16bf)__A, (__v16bf)__B,
856+
(__v16bf)__C);
857857
}
858858

859859
static __inline__ __m256bh __DEFAULT_FN_ATTRS256
@@ -880,8 +880,8 @@ static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_maskz_fmadd_pbh(
880880

881881
static __inline__ __m256bh __DEFAULT_FN_ATTRS256
882882
_mm256_fmsub_pbh(__m256bh __A, __m256bh __B, __m256bh __C) {
883-
return (__m256bh)__builtin_ia32_vfmaddbf16256((__v16bf)__A, (__v16bf)__B,
884-
-(__v16bf)__C);
883+
return (__m256bh)__builtin_elementwise_fma((__v16bf)__A, (__v16bf)__B,
884+
-(__v16bf)__C);
885885
}
886886

887887
static __inline__ __m256bh __DEFAULT_FN_ATTRS256
@@ -908,8 +908,8 @@ static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_maskz_fmsub_pbh(
908908

909909
static __inline__ __m256bh __DEFAULT_FN_ATTRS256
910910
_mm256_fnmadd_pbh(__m256bh __A, __m256bh __B, __m256bh __C) {
911-
return (__m256bh)__builtin_ia32_vfmaddbf16256((__v16bf)__A, -(__v16bf)__B,
912-
(__v16bf)__C);
911+
return (__m256bh)__builtin_elementwise_fma((__v16bf)__A, -(__v16bf)__B,
912+
(__v16bf)__C);
913913
}
914914

915915
static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_mask_fnmadd_pbh(
@@ -938,8 +938,8 @@ static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_maskz_fnmadd_pbh(
938938

939939
static __inline__ __m256bh __DEFAULT_FN_ATTRS256
940940
_mm256_fnmsub_pbh(__m256bh __A, __m256bh __B, __m256bh __C) {
941-
return (__m256bh)__builtin_ia32_vfmaddbf16256((__v16bf)__A, -(__v16bf)__B,
942-
-(__v16bf)__C);
941+
return (__m256bh)__builtin_elementwise_fma((__v16bf)__A, -(__v16bf)__B,
942+
-(__v16bf)__C);
943943
}
944944

945945
static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_mask_fnmsub_pbh(
@@ -969,8 +969,8 @@ static __inline__ __m256bh __DEFAULT_FN_ATTRS256 _mm256_maskz_fnmsub_pbh(
969969
static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_fmadd_pbh(__m128bh __A,
970970
__m128bh __B,
971971
__m128bh __C) {
972-
return (__m128bh)__builtin_ia32_vfmaddbf16128((__v8bf)__A, (__v8bf)__B,
973-
(__v8bf)__C);
972+
return (__m128bh)__builtin_elementwise_fma((__v8bf)__A, (__v8bf)__B,
973+
(__v8bf)__C);
974974
}
975975

976976
static __inline__ __m128bh __DEFAULT_FN_ATTRS128
@@ -997,8 +997,8 @@ _mm_maskz_fmadd_pbh(__mmask8 __U, __m128bh __A, __m128bh __B, __m128bh __C) {
997997
static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_fmsub_pbh(__m128bh __A,
998998
__m128bh __B,
999999
__m128bh __C) {
1000-
return (__m128bh)__builtin_ia32_vfmaddbf16128((__v8bf)__A, (__v8bf)__B,
1001-
-(__v8bf)__C);
1000+
return (__m128bh)__builtin_elementwise_fma((__v8bf)__A, (__v8bf)__B,
1001+
-(__v8bf)__C);
10021002
}
10031003

10041004
static __inline__ __m128bh __DEFAULT_FN_ATTRS128
@@ -1025,8 +1025,8 @@ _mm_maskz_fmsub_pbh(__mmask8 __U, __m128bh __A, __m128bh __B, __m128bh __C) {
10251025
static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_fnmadd_pbh(__m128bh __A,
10261026
__m128bh __B,
10271027
__m128bh __C) {
1028-
return (__m128bh)__builtin_ia32_vfmaddbf16128((__v8bf)__A, -(__v8bf)__B,
1029-
(__v8bf)__C);
1028+
return (__m128bh)__builtin_elementwise_fma((__v8bf)__A, -(__v8bf)__B,
1029+
(__v8bf)__C);
10301030
}
10311031

10321032
static __inline__ __m128bh __DEFAULT_FN_ATTRS128
@@ -1053,8 +1053,8 @@ _mm_maskz_fnmadd_pbh(__mmask8 __U, __m128bh __A, __m128bh __B, __m128bh __C) {
10531053
static __inline__ __m128bh __DEFAULT_FN_ATTRS128 _mm_fnmsub_pbh(__m128bh __A,
10541054
__m128bh __B,
10551055
__m128bh __C) {
1056-
return (__m128bh)__builtin_ia32_vfmaddbf16128((__v8bf)__A, -(__v8bf)__B,
1057-
-(__v8bf)__C);
1056+
return (__m128bh)__builtin_elementwise_fma((__v8bf)__A, -(__v8bf)__B,
1057+
-(__v8bf)__C);
10581058
}
10591059

10601060
static __inline__ __m128bh __DEFAULT_FN_ATTRS128

0 commit comments

Comments
 (0)