Skip to content

[Headers][X86] Update AVX/AVX512 float/double add/sub/mul/div/unpck intrinsics to be used in constexpr #152435

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Aug 7, 2025
Merged

[Headers][X86] Update AVX/AVX512 float/double add/sub/mul/div/unpck intrinsics to be used in constexpr #152435

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7127a51
[Headers][X86] Update AVX/AVX512 float/double add/sub/mul/div/unpck i…
yichi170 Aug 6, 2025
bb0c706
Format and update tests
yichi170 Aug 7, 2025
f3ff976
Merge branch 'main' into avx-constexpr
RKSimon Aug 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 26 additions & 39 deletions clang/lib/Headers/avx512fintrin.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -835,45 +835,38 @@ _mm512_xor_si512(__m512i __a, __m512i __b)

/* Arithmetic */

static __inline __m512d __DEFAULT_FN_ATTRS512
_mm512_add_pd(__m512d __a, __m512d __b)
{
static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
_mm512_add_pd(__m512d __a, __m512d __b) {
return (__m512d)((__v8df)__a + (__v8df)__b);
}

static __inline __m512 __DEFAULT_FN_ATTRS512
_mm512_add_ps(__m512 __a, __m512 __b)
{
static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
_mm512_add_ps(__m512 __a, __m512 __b) {
return (__m512)((__v16sf)__a + (__v16sf)__b);
}

static __inline __m512d __DEFAULT_FN_ATTRS512
_mm512_mul_pd(__m512d __a, __m512d __b)
{
static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
_mm512_mul_pd(__m512d __a, __m512d __b) {
return (__m512d)((__v8df)__a * (__v8df)__b);
}

static __inline __m512 __DEFAULT_FN_ATTRS512
_mm512_mul_ps(__m512 __a, __m512 __b)
{
static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
_mm512_mul_ps(__m512 __a, __m512 __b) {
return (__m512)((__v16sf)__a * (__v16sf)__b);
}

static __inline __m512d __DEFAULT_FN_ATTRS512
_mm512_sub_pd(__m512d __a, __m512d __b)
{
static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
_mm512_sub_pd(__m512d __a, __m512d __b) {
return (__m512d)((__v8df)__a - (__v8df)__b);
}

static __inline __m512 __DEFAULT_FN_ATTRS512
_mm512_sub_ps(__m512 __a, __m512 __b)
{
static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
_mm512_sub_ps(__m512 __a, __m512 __b) {
return (__m512)((__v16sf)__a - (__v16sf)__b);
}

static __inline__ __m512i __DEFAULT_FN_ATTRS512
_mm512_add_epi64 (__m512i __A, __m512i __B)
{
static __inline__ __m512i __DEFAULT_FN_ATTRS512_CONSTEXPR
_mm512_add_epi64(__m512i __A, __m512i __B) {
return (__m512i) ((__v8du) __A + (__v8du) __B);
}

Expand Down Expand Up @@ -2315,9 +2308,8 @@ _mm_maskz_div_sd(__mmask8 __U,__m128d __A, __m128d __B) {
(__v2df)_mm_setzero_pd(), \
(__mmask8)(U), (int)(R)))

static __inline __m512d __DEFAULT_FN_ATTRS512
_mm512_div_pd(__m512d __a, __m512d __b)
{
static __inline __m512d
__DEFAULT_FN_ATTRS512_CONSTEXPR _mm512_div_pd(__m512d __a, __m512d __b) {
return (__m512d)((__v8df)__a/(__v8df)__b);
}

Expand All @@ -2335,9 +2327,8 @@ _mm512_maskz_div_pd(__mmask8 __U, __m512d __A, __m512d __B) {
(__v8df)_mm512_setzero_pd());
}

static __inline __m512 __DEFAULT_FN_ATTRS512
_mm512_div_ps(__m512 __a, __m512 __b)
{
static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
_mm512_div_ps(__m512 __a, __m512 __b) {
return (__m512)((__v16sf)__a/(__v16sf)__b);
}

Expand Down Expand Up @@ -4123,9 +4114,8 @@ _mm512_cvtss_f32(__m512 __a)

/* Unpack and Interleave */

static __inline __m512d __DEFAULT_FN_ATTRS512
_mm512_unpackhi_pd(__m512d __a, __m512d __b)
{
static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
_mm512_unpackhi_pd(__m512d __a, __m512d __b) {
return (__m512d)__builtin_shufflevector((__v8df)__a, (__v8df)__b,
1, 9, 1+2, 9+2, 1+4, 9+4, 1+6, 9+6);
}
Expand All @@ -4146,9 +4136,8 @@ _mm512_maskz_unpackhi_pd(__mmask8 __U, __m512d __A, __m512d __B)
(__v8df)_mm512_setzero_pd());
}

static __inline __m512d __DEFAULT_FN_ATTRS512
_mm512_unpacklo_pd(__m512d __a, __m512d __b)
{
static __inline __m512d __DEFAULT_FN_ATTRS512_CONSTEXPR
_mm512_unpacklo_pd(__m512d __a, __m512d __b) {
return (__m512d)__builtin_shufflevector((__v8df)__a, (__v8df)__b,
0, 8, 0+2, 8+2, 0+4, 8+4, 0+6, 8+6);
}
Expand All @@ -4169,9 +4158,8 @@ _mm512_maskz_unpacklo_pd (__mmask8 __U, __m512d __A, __m512d __B)
(__v8df)_mm512_setzero_pd());
}

static __inline __m512 __DEFAULT_FN_ATTRS512
_mm512_unpackhi_ps(__m512 __a, __m512 __b)
{
static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
_mm512_unpackhi_ps(__m512 __a, __m512 __b) {
return (__m512)__builtin_shufflevector((__v16sf)__a, (__v16sf)__b,
2, 18, 3, 19,
2+4, 18+4, 3+4, 19+4,
Expand All @@ -4195,9 +4183,8 @@ _mm512_maskz_unpackhi_ps (__mmask16 __U, __m512 __A, __m512 __B)
(__v16sf)_mm512_setzero_ps());
}

static __inline __m512 __DEFAULT_FN_ATTRS512
_mm512_unpacklo_ps(__m512 __a, __m512 __b)
{
static __inline __m512 __DEFAULT_FN_ATTRS512_CONSTEXPR
_mm512_unpacklo_ps(__m512 __a, __m512 __b) {
return (__m512)__builtin_shufflevector((__v16sf)__a, (__v16sf)__b,
0, 16, 1, 17,
0+4, 16+4, 1+4, 17+4,
Expand Down
60 changes: 24 additions & 36 deletions clang/lib/Headers/avxintrin.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,9 +87,8 @@ typedef __bf16 __m256bh __attribute__((__vector_size__(32), __aligned__(32)));
/// A 256-bit vector of [4 x double] containing one of the source operands.
/// \returns A 256-bit vector of [4 x double] containing the sums of both
/// operands.
static __inline __m256d __DEFAULT_FN_ATTRS
_mm256_add_pd(__m256d __a, __m256d __b)
{
static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
_mm256_add_pd(__m256d __a, __m256d __b) {
return (__m256d)((__v4df)__a+(__v4df)__b);
}

Expand All @@ -105,9 +104,8 @@ _mm256_add_pd(__m256d __a, __m256d __b)
/// A 256-bit vector of [8 x float] containing one of the source operands.
/// \returns A 256-bit vector of [8 x float] containing the sums of both
/// operands.
static __inline __m256 __DEFAULT_FN_ATTRS
_mm256_add_ps(__m256 __a, __m256 __b)
{
static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_add_ps(__m256 __a,
__m256 __b) {
return (__m256)((__v8sf)__a+(__v8sf)__b);
}

Expand All @@ -123,9 +121,8 @@ _mm256_add_ps(__m256 __a, __m256 __b)
/// A 256-bit vector of [4 x double] containing the subtrahend.
/// \returns A 256-bit vector of [4 x double] containing the differences between
/// both operands.
static __inline __m256d __DEFAULT_FN_ATTRS
_mm256_sub_pd(__m256d __a, __m256d __b)
{
static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
_mm256_sub_pd(__m256d __a, __m256d __b) {
return (__m256d)((__v4df)__a-(__v4df)__b);
}

Expand All @@ -141,9 +138,8 @@ _mm256_sub_pd(__m256d __a, __m256d __b)
/// A 256-bit vector of [8 x float] containing the subtrahend.
/// \returns A 256-bit vector of [8 x float] containing the differences between
/// both operands.
static __inline __m256 __DEFAULT_FN_ATTRS
_mm256_sub_ps(__m256 __a, __m256 __b)
{
static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_sub_ps(__m256 __a,
__m256 __b) {
return (__m256)((__v8sf)__a-(__v8sf)__b);
}

Expand Down Expand Up @@ -197,9 +193,8 @@ _mm256_addsub_ps(__m256 __a, __m256 __b)
/// A 256-bit vector of [4 x double] containing the divisor.
/// \returns A 256-bit vector of [4 x double] containing the quotients of both
/// operands.
static __inline __m256d __DEFAULT_FN_ATTRS
_mm256_div_pd(__m256d __a, __m256d __b)
{
static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
_mm256_div_pd(__m256d __a, __m256d __b) {
return (__m256d)((__v4df)__a/(__v4df)__b);
}

Expand All @@ -215,9 +210,8 @@ _mm256_div_pd(__m256d __a, __m256d __b)
/// A 256-bit vector of [8 x float] containing the divisor.
/// \returns A 256-bit vector of [8 x float] containing the quotients of both
/// operands.
static __inline __m256 __DEFAULT_FN_ATTRS
_mm256_div_ps(__m256 __a, __m256 __b)
{
static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_div_ps(__m256 __a,
__m256 __b) {
return (__m256)((__v8sf)__a/(__v8sf)__b);
}

Expand Down Expand Up @@ -317,9 +311,8 @@ _mm256_min_ps(__m256 __a, __m256 __b)
/// A 256-bit vector of [4 x double] containing one of the operands.
/// \returns A 256-bit vector of [4 x double] containing the products of both
/// operands.
static __inline __m256d __DEFAULT_FN_ATTRS
_mm256_mul_pd(__m256d __a, __m256d __b)
{
static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
_mm256_mul_pd(__m256d __a, __m256d __b) {
return (__m256d)((__v4df)__a * (__v4df)__b);
}

Expand All @@ -335,9 +328,8 @@ _mm256_mul_pd(__m256d __a, __m256d __b)
/// A 256-bit vector of [8 x float] containing one of the operands.
/// \returns A 256-bit vector of [8 x float] containing the products of both
/// operands.
static __inline __m256 __DEFAULT_FN_ATTRS
_mm256_mul_ps(__m256 __a, __m256 __b)
{
static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR _mm256_mul_ps(__m256 __a,
__m256 __b) {
return (__m256)((__v8sf)__a * (__v8sf)__b);
}

Expand Down Expand Up @@ -2462,9 +2454,8 @@ _mm256_movedup_pd(__m256d __a)
/// Bits [127:64] are written to bits [127:64] of the return value. \n
/// Bits [255:192] are written to bits [255:192] of the return value. \n
/// \returns A 256-bit vector of [4 x double] containing the interleaved values.
static __inline __m256d __DEFAULT_FN_ATTRS
_mm256_unpackhi_pd(__m256d __a, __m256d __b)
{
static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
_mm256_unpackhi_pd(__m256d __a, __m256d __b) {
return __builtin_shufflevector((__v4df)__a, (__v4df)__b, 1, 5, 1+2, 5+2);
}

Expand All @@ -2484,9 +2475,8 @@ _mm256_unpackhi_pd(__m256d __a, __m256d __b)
/// Bits [63:0] are written to bits [127:64] of the return value. \n
/// Bits [191:128] are written to bits [255:192] of the return value. \n
/// \returns A 256-bit vector of [4 x double] containing the interleaved values.
static __inline __m256d __DEFAULT_FN_ATTRS
_mm256_unpacklo_pd(__m256d __a, __m256d __b)
{
static __inline __m256d __DEFAULT_FN_ATTRS_CONSTEXPR
_mm256_unpacklo_pd(__m256d __a, __m256d __b) {
return __builtin_shufflevector((__v4df)__a, (__v4df)__b, 0, 4, 0+2, 4+2);
}

Expand All @@ -2511,9 +2501,8 @@ _mm256_unpacklo_pd(__m256d __a, __m256d __b)
/// Bits [223:192] are written to bits [191:160] of the return value. \n
/// Bits [255:224] are written to bits [255:224] of the return value.
/// \returns A 256-bit vector of [8 x float] containing the interleaved values.
static __inline __m256 __DEFAULT_FN_ATTRS
_mm256_unpackhi_ps(__m256 __a, __m256 __b)
{
static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
_mm256_unpackhi_ps(__m256 __a, __m256 __b) {
return __builtin_shufflevector((__v8sf)__a, (__v8sf)__b, 2, 10, 2+1, 10+1, 6, 14, 6+1, 14+1);
}

Expand All @@ -2538,9 +2527,8 @@ _mm256_unpackhi_ps(__m256 __a, __m256 __b)
/// Bits [159:128] are written to bits [191:160] of the return value. \n
/// Bits [191:160] are written to bits [255:224] of the return value.
/// \returns A 256-bit vector of [8 x float] containing the interleaved values.
static __inline __m256 __DEFAULT_FN_ATTRS
_mm256_unpacklo_ps(__m256 __a, __m256 __b)
{
static __inline __m256 __DEFAULT_FN_ATTRS_CONSTEXPR
_mm256_unpacklo_ps(__m256 __a, __m256 __b) {
return __builtin_shufflevector((__v8sf)__a, (__v8sf)__b, 0, 8, 0+1, 8+1, 4, 12, 4+1, 12+1);
}

Expand Down
12 changes: 12 additions & 0 deletions clang/test/CodeGen/X86/avx-builtins.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,14 @@ __m256d test_mm256_add_pd(__m256d A, __m256d B) {
// CHECK: fadd <4 x double>
return _mm256_add_pd(A, B);
}
TEST_CONSTEXPR(match_m256d( _mm256_add_pd((__m256d){-4.0, -5.0, +6.0, +7.0}, (__m256d){-4.0, -5.0, +6.0, +7.0}), -8.0, -10.0, +12.0, +14.0));

__m256 test_mm256_add_ps(__m256 A, __m256 B) {
// CHECK-LABEL: test_mm256_add_ps
// CHECK: fadd <8 x float>
return _mm256_add_ps(A, B);
}
TEST_CONSTEXPR(match_m256(_mm256_add_ps((__m256){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m256){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}), -8.0f, -10.0f, +12.0f, +14.0f, +14.0f, +12.0f, -10.0f, -8.0f));

__m256d test_mm256_addsub_pd(__m256d A, __m256d B) {
// CHECK-LABEL: test_mm256_addsub_pd
Expand Down Expand Up @@ -977,12 +979,14 @@ __m256d test_mm256_div_pd(__m256d A, __m256d B) {
// CHECK: fdiv <4 x double>
return _mm256_div_pd(A, B);
}
TEST_CONSTEXPR(match_m256d( _mm256_div_pd((__m256d){-4.0, -5.0, +6.0, +7.0}, (__m256d){-1.0, +1.0, -1.0, +1.0}), +4.0, -5.0, -6.0, +7.0));

__m256 test_mm256_div_ps(__m256 A, __m256 B) {
// CHECK-LABEL: test_mm256_div_ps
// CHECK: fdiv <8 x float>
return _mm256_div_ps(A, B);
}
TEST_CONSTEXPR(match_m256( _mm256_div_ps((__m256){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m256){-1.0f, +1.0f, -1.0f, +1.0f, +1.0f, -1.0f, +1.0f, -1.0f}), +4.0f, -5.0f, -6.0f, +7.0f, +7.0f, -6.0f, -5.0f, +4.0f));

__m256 test_mm256_dp_ps(__m256 A, __m256 B) {
// CHECK-LABEL: test_mm256_dp_ps
Expand Down Expand Up @@ -1295,12 +1299,14 @@ __m256d test_mm256_mul_pd(__m256d A, __m256d B) {
// CHECK: fmul <4 x double>
return _mm256_mul_pd(A, B);
}
TEST_CONSTEXPR(match_m256d( _mm256_mul_pd((__m256d){-4.0, -5.0, +6.0, +7.0}, (__m256d){-4.0, -5.0, +6.0, +7.0}), +16.0, +25.0, +36.0, +49.0));

__m256 test_mm256_mul_ps(__m256 A, __m256 B) {
// CHECK-LABEL: test_mm256_mul_ps
// CHECK: fmul <8 x float>
return _mm256_mul_ps(A, B);
}
TEST_CONSTEXPR(match_m256( _mm256_mul_ps((__m256){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m256){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}), +16.0f, +25.0f, +36.0f, +49.0f, +49.0f, +36.0f, +25.0f, +16.0f));

__m256d test_mm256_or_pd(__m256d A, __m256d B) {
// CHECK-LABEL: test_mm256_or_pd
Expand Down Expand Up @@ -1933,12 +1939,14 @@ __m256d test_mm256_sub_pd(__m256d A, __m256d B) {
// CHECK: fsub <4 x double>
return _mm256_sub_pd(A, B);
}
TEST_CONSTEXPR(match_m256d( _mm256_sub_pd((__m256d){-4.0, -5.0, +6.0, +7.0}, (__m256d){-0.0, +0.0, +2.0, -1.0}), -4.0, -5.0, 4.0, 8.0));

__m256 test_mm256_sub_ps(__m256 A, __m256 B) {
// CHECK-LABEL: test_mm256_sub_ps
// CHECK: fsub <8 x float>
return _mm256_sub_ps(A, B);
}
TEST_CONSTEXPR(match_m256( _mm256_sub_ps((__m256){-4.0f, -5.0f, +6.0f, +7.0f, +7.0f, +6.0f, -5.0f, -4.0f}, (__m256){-0.0f, +0.0f, +2.0f, -1.0f, -1.0f, +2.0f, +0.0f, -0.0f}), -4.0f, -5.0f, 4.0f, 8.0f, 8.0f, 4.0f, -5.0f, -4.0f));

int test_mm_testc_pd(__m128d A, __m128d B) {
// CHECK-LABEL: test_mm_testc_pd
Expand Down Expand Up @@ -2062,24 +2070,28 @@ __m256d test_mm256_unpackhi_pd(__m256d A, __m256d B) {
// CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
return _mm256_unpackhi_pd(A, B);
}
TEST_CONSTEXPR(match_m256d(_mm256_unpackhi_pd((__m256d){+1.0, +2.0, +3.0, +4.0}, (__m256d){+5.0, +6.0, +7.0, +8.0}), +2.0, +6.0, +4.0, +8.0));

__m256 test_mm256_unpackhi_ps(__m256 A, __m256 B) {
// CHECK-LABEL: test_mm256_unpackhi_ps
// CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
return _mm256_unpackhi_ps(A, B);
}
TEST_CONSTEXPR(match_m256(_mm256_unpackhi_ps((__m256){+0.0f, +1.0f, +2.0f, +3.0f, +4.0f, +5.0f, +6.0f, +7.0f}, (__m256){+10.0f, +11.0f, +12.0f, +13.0f, +14.0f, +15.0f, +16.0f, +17.0f}), +2.0f, +12.0f, +3.0f, +13.0f, +6.0f, +16.0f, +7.0f, +17.0f));

__m256d test_mm256_unpacklo_pd(__m256d A, __m256d B) {
// CHECK-LABEL: test_mm256_unpacklo_pd
// CHECK: shufflevector <4 x double> %{{.*}}, <4 x double> %{{.*}}, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
return _mm256_unpacklo_pd(A, B);
}
TEST_CONSTEXPR(match_m256d(_mm256_unpacklo_pd((__m256d){+1.0, +2.0, +3.0, +4.0}, (__m256d){+5.0, +6.0, +7.0, +8.0}), +1.0, +5.0, +3.0, +7.0));

__m256 test_mm256_unpacklo_ps(__m256 A, __m256 B) {
// CHECK-LABEL: test_mm256_unpacklo_ps
// CHECK: shufflevector <8 x float> %{{.*}}, <8 x float> %{{.*}}, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
return _mm256_unpacklo_ps(A, B);
}
TEST_CONSTEXPR(match_m256(_mm256_unpacklo_ps((__m256){+0.0f, +1.0f, +2.0f, +3.0f, +4.0f, +5.0f, +6.0f, +7.0f}, (__m256){+10.0f, +11.0f, +12.0f, +13.0f, +14.0f, +15.0f, +16.0f, +17.0f}), +0.0f, +10.0f, +1.0f, +11.0f, +4.0f, +14.0f, +5.0f, +15.0f));

__m256d test_mm256_xor_pd(__m256d A, __m256d B) {
// CHECK-LABEL: test_mm256_xor_pd
Expand Down
Loading
Loading