Skip to content

[X86][Clang] VectorExprEvaluator::VisitCallExpr / InterpretBuiltin - Allow AVX/AVX512 IFMA madd52 intrinsics to be used in constexpr #161056

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Oct 14, 2025
Merged

[X86][Clang] VectorExprEvaluator::VisitCallExpr / InterpretBuiltin - Allow AVX/AVX512 IFMA madd52 intrinsics to be used in constexpr #161056

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1a0a6e4
[X86][Clang] VectorExprEvaluator::VisitCallExpr / InterpretBuiltin - …
chaitanyav Sep 28, 2025
78abc73
Merge branch 'main' into issue_160498
RKSimon Oct 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 4 additions & 10 deletions clang/include/clang/Basic/BuiltinsX86.td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2137,24 +2137,18 @@ let Features = "avx512vl", Attributes = [NoThrow, RequiredVectorWidth<256>] in {
def movdqa64store256_mask : X86Builtin<"void(_Vector<4, long long int *>, _Vector<4, long long int>, unsigned char)">;
}

let Features = "avx512ifma", Attributes = [NoThrow, Const, RequiredVectorWidth<512>] in {
let Features = "avx512ifma", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<512>] in {
def vpmadd52huq512 : X86Builtin<"_Vector<8, long long int>(_Vector<8, long long int>, _Vector<8, long long int>, _Vector<8, long long int>)">;
def vpmadd52luq512 : X86Builtin<"_Vector<8, long long int>(_Vector<8, long long int>, _Vector<8, long long int>, _Vector<8, long long int>)">;
}

let Features = "avx512ifma,avx512vl|avxifma", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
let Features = "avx512ifma,avx512vl|avxifma", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<128>] in {
def vpmadd52huq128 : X86Builtin<"_Vector<2, long long int>(_Vector<2, long long int>, _Vector<2, long long int>, _Vector<2, long long int>)">;
}

let Features = "avx512ifma,avx512vl|avxifma", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
def vpmadd52huq256 : X86Builtin<"_Vector<4, long long int>(_Vector<4, long long int>, _Vector<4, long long int>, _Vector<4, long long int>)">;
}

let Features = "avx512ifma,avx512vl|avxifma", Attributes = [NoThrow, Const, RequiredVectorWidth<128>] in {
def vpmadd52luq128 : X86Builtin<"_Vector<2, long long int>(_Vector<2, long long int>, _Vector<2, long long int>, _Vector<2, long long int>)">;
}

let Features = "avx512ifma,avx512vl|avxifma", Attributes = [NoThrow, Const, RequiredVectorWidth<256>] in {
let Features = "avx512ifma,avx512vl|avxifma", Attributes = [NoThrow, Const, Constexpr, RequiredVectorWidth<256>] in {
def vpmadd52huq256 : X86Builtin<"_Vector<4, long long int>(_Vector<4, long long int>, _Vector<4, long long int>, _Vector<4, long long int>)">;
def vpmadd52luq256 : X86Builtin<"_Vector<4, long long int>(_Vector<4, long long int>, _Vector<4, long long int>, _Vector<4, long long int>)">;
}

Expand Down
15 changes: 15 additions & 0 deletions clang/lib/AST/ByteCode/InterpBuiltin.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3818,6 +3818,21 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
return F;
});

case X86::BI__builtin_ia32_vpmadd52luq128:
case X86::BI__builtin_ia32_vpmadd52luq256:
case X86::BI__builtin_ia32_vpmadd52luq512:
return interp__builtin_elementwise_triop(
S, OpPC, Call, [](const APSInt &A, const APSInt &B, const APSInt &C) {
return A + (B.trunc(52) * C.trunc(52)).zext(64);
});
case X86::BI__builtin_ia32_vpmadd52huq128:
case X86::BI__builtin_ia32_vpmadd52huq256:
case X86::BI__builtin_ia32_vpmadd52huq512:
return interp__builtin_elementwise_triop(
S, OpPC, Call, [](const APSInt &A, const APSInt &B, const APSInt &C) {
return A + llvm::APIntOps::mulhu(B.trunc(52), C.trunc(52)).zext(64);
});

case X86::BI__builtin_ia32_vpshldd128:
case X86::BI__builtin_ia32_vpshldd256:
case X86::BI__builtin_ia32_vpshldd512:
Expand Down
48 changes: 48 additions & 0 deletions clang/lib/AST/ExprConstant.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11974,6 +11974,54 @@ bool VectorExprEvaluator::VisitCallExpr(const CallExpr *E) {

return Success(APValue(ResultElements.data(), ResultElements.size()), E);
}

case X86::BI__builtin_ia32_vpmadd52luq128:
case X86::BI__builtin_ia32_vpmadd52luq256:
case X86::BI__builtin_ia32_vpmadd52luq512: {
APValue A, B, C;
if (!EvaluateAsRValue(Info, E->getArg(0), A) ||
!EvaluateAsRValue(Info, E->getArg(1), B) ||
!EvaluateAsRValue(Info, E->getArg(2), C))
return false;

unsigned ALen = A.getVectorLength();
SmallVector<APValue, 4> ResultElements;
ResultElements.reserve(ALen);

for (unsigned EltNum = 0; EltNum < ALen; EltNum += 1) {
APInt AElt = A.getVectorElt(EltNum).getInt();
APInt BElt = B.getVectorElt(EltNum).getInt().trunc(52);
APInt CElt = C.getVectorElt(EltNum).getInt().trunc(52);
APSInt ResElt(AElt + (BElt * CElt).zext(64), false);
ResultElements.push_back(APValue(ResElt));
}

return Success(APValue(ResultElements.data(), ResultElements.size()), E);
}
case X86::BI__builtin_ia32_vpmadd52huq128:
case X86::BI__builtin_ia32_vpmadd52huq256:
case X86::BI__builtin_ia32_vpmadd52huq512: {
APValue A, B, C;
if (!EvaluateAsRValue(Info, E->getArg(0), A) ||
!EvaluateAsRValue(Info, E->getArg(1), B) ||
!EvaluateAsRValue(Info, E->getArg(2), C))
return false;

unsigned ALen = A.getVectorLength();
SmallVector<APValue, 4> ResultElements;
ResultElements.reserve(ALen);

for (unsigned EltNum = 0; EltNum < ALen; EltNum += 1) {
APInt AElt = A.getVectorElt(EltNum).getInt();
APInt BElt = B.getVectorElt(EltNum).getInt().trunc(52);
APInt CElt = C.getVectorElt(EltNum).getInt().trunc(52);
APSInt ResElt(AElt + llvm::APIntOps::mulhu(BElt, CElt).zext(64), false);
ResultElements.push_back(APValue(ResElt));
}

return Success(APValue(ResultElements.data(), ResultElements.size()), E);
}

case clang::X86::BI__builtin_ia32_vprotbi:
case clang::X86::BI__builtin_ia32_vprotdi:
case clang::X86::BI__builtin_ia32_vprotqi:
Expand Down
63 changes: 31 additions & 32 deletions clang/lib/Headers/avx512ifmaintrin.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,54 +15,53 @@
#define __IFMAINTRIN_H

/* Define the default attributes for the functions in this file. */
#if defined(__cplusplus) && (__cplusplus >= 201103L)
#define __DEFAULT_FN_ATTRS \
constexpr \
__attribute__((__always_inline__, __nodebug__, __target__("avx512ifma"), \
__min_vector_width__(512)))
#else
#define __DEFAULT_FN_ATTRS \
__attribute__((__always_inline__, __nodebug__, __target__("avx512ifma"), \
__min_vector_width__(512)))
#endif

static __inline__ __m512i __DEFAULT_FN_ATTRS
_mm512_madd52hi_epu64 (__m512i __X, __m512i __Y, __m512i __Z)
{
return (__m512i)__builtin_ia32_vpmadd52huq512((__v8di) __X, (__v8di) __Y,
(__v8di) __Z);
_mm512_madd52hi_epu64(__m512i __X, __m512i __Y, __m512i __Z) {
return (__m512i)__builtin_ia32_vpmadd52huq512((__v8di)__X, (__v8di)__Y,
(__v8di)__Z);
}

static __inline__ __m512i __DEFAULT_FN_ATTRS
_mm512_mask_madd52hi_epu64 (__m512i __W, __mmask8 __M, __m512i __X, __m512i __Y)
{
return (__m512i)__builtin_ia32_selectq_512(__M,
(__v8di)_mm512_madd52hi_epu64(__W, __X, __Y),
(__v8di)__W);
static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_madd52hi_epu64(
__m512i __W, __mmask8 __M, __m512i __X, __m512i __Y) {
return (__m512i)__builtin_ia32_selectq_512(
__M, (__v8di)_mm512_madd52hi_epu64(__W, __X, __Y), (__v8di)__W);
}

static __inline__ __m512i __DEFAULT_FN_ATTRS
_mm512_maskz_madd52hi_epu64 (__mmask8 __M, __m512i __X, __m512i __Y, __m512i __Z)
{
return (__m512i)__builtin_ia32_selectq_512(__M,
(__v8di)_mm512_madd52hi_epu64(__X, __Y, __Z),
(__v8di)_mm512_setzero_si512());
static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_madd52hi_epu64(
__mmask8 __M, __m512i __X, __m512i __Y, __m512i __Z) {
return (__m512i)__builtin_ia32_selectq_512(
__M, (__v8di)_mm512_madd52hi_epu64(__X, __Y, __Z),
(__v8di)_mm512_setzero_si512());
}

static __inline__ __m512i __DEFAULT_FN_ATTRS
_mm512_madd52lo_epu64 (__m512i __X, __m512i __Y, __m512i __Z)
{
return (__m512i)__builtin_ia32_vpmadd52luq512((__v8di) __X, (__v8di) __Y,
(__v8di) __Z);
_mm512_madd52lo_epu64(__m512i __X, __m512i __Y, __m512i __Z) {
return (__m512i)__builtin_ia32_vpmadd52luq512((__v8di)__X, (__v8di)__Y,
(__v8di)__Z);
}

static __inline__ __m512i __DEFAULT_FN_ATTRS
_mm512_mask_madd52lo_epu64 (__m512i __W, __mmask8 __M, __m512i __X, __m512i __Y)
{
return (__m512i)__builtin_ia32_selectq_512(__M,
(__v8di)_mm512_madd52lo_epu64(__W, __X, __Y),
(__v8di)__W);
static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_mask_madd52lo_epu64(
__m512i __W, __mmask8 __M, __m512i __X, __m512i __Y) {
return (__m512i)__builtin_ia32_selectq_512(
__M, (__v8di)_mm512_madd52lo_epu64(__W, __X, __Y), (__v8di)__W);
}

static __inline__ __m512i __DEFAULT_FN_ATTRS
_mm512_maskz_madd52lo_epu64 (__mmask8 __M, __m512i __X, __m512i __Y, __m512i __Z)
{
return (__m512i)__builtin_ia32_selectq_512(__M,
(__v8di)_mm512_madd52lo_epu64(__X, __Y, __Z),
(__v8di)_mm512_setzero_si512());
static __inline__ __m512i __DEFAULT_FN_ATTRS _mm512_maskz_madd52lo_epu64(
__mmask8 __M, __m512i __X, __m512i __Y, __m512i __Z) {
return (__m512i)__builtin_ia32_selectq_512(
__M, (__v8di)_mm512_madd52lo_epu64(__X, __Y, __Z),
(__v8di)_mm512_setzero_si512());
}

#undef __DEFAULT_FN_ATTRS
Expand Down
92 changes: 46 additions & 46 deletions clang/lib/Headers/avx512ifmavlintrin.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,24 @@
*===-----------------------------------------------------------------------===
*/
#ifndef __IMMINTRIN_H
#error "Never use <avx512ifmavlintrin.h> directly; include <immintrin.h> instead."
#error \
"Never use <avx512ifmavlintrin.h> directly; include <immintrin.h> instead."
#endif

#ifndef __IFMAVLINTRIN_H
#define __IFMAVLINTRIN_H

/* Define the default attributes for the functions in this file. */
#if defined(__cplusplus) && (__cplusplus >= 201103L)
#define __DEFAULT_FN_ATTRS128 \
constexpr __attribute__((__always_inline__, __nodebug__, \
__target__("avx512ifma,avx512vl"), \
__min_vector_width__(128)))
#define __DEFAULT_FN_ATTRS256 \
constexpr __attribute__((__always_inline__, __nodebug__, \
__target__("avx512ifma,avx512vl"), \
__min_vector_width__(256)))
#else
#define __DEFAULT_FN_ATTRS128 \
__attribute__((__always_inline__, __nodebug__, \
__target__("avx512ifma,avx512vl"), \
Expand All @@ -24,6 +35,8 @@
__target__("avx512ifma,avx512vl"), \
__min_vector_width__(256)))

#endif

#define _mm_madd52hi_epu64(X, Y, Z) \
((__m128i)__builtin_ia32_vpmadd52huq128((__v2di)(X), (__v2di)(Y), \
(__v2di)(Z)))
Expand All @@ -41,70 +54,57 @@
(__v4di)(Z)))

static __inline__ __m128i __DEFAULT_FN_ATTRS128
_mm_mask_madd52hi_epu64 (__m128i __W, __mmask8 __M, __m128i __X, __m128i __Y)
{
return (__m128i)__builtin_ia32_selectq_128(__M,
(__v2di)_mm_madd52hi_epu64(__W, __X, __Y),
(__v2di)__W);
_mm_mask_madd52hi_epu64(__m128i __W, __mmask8 __M, __m128i __X, __m128i __Y) {
return (__m128i)__builtin_ia32_selectq_128(
__M, (__v2di)_mm_madd52hi_epu64(__W, __X, __Y), (__v2di)__W);
}

static __inline__ __m128i __DEFAULT_FN_ATTRS128
_mm_maskz_madd52hi_epu64 (__mmask8 __M, __m128i __X, __m128i __Y, __m128i __Z)
{
return (__m128i)__builtin_ia32_selectq_128(__M,
(__v2di)_mm_madd52hi_epu64(__X, __Y, __Z),
(__v2di)_mm_setzero_si128());
_mm_maskz_madd52hi_epu64(__mmask8 __M, __m128i __X, __m128i __Y, __m128i __Z) {
return (__m128i)__builtin_ia32_selectq_128(
__M, (__v2di)_mm_madd52hi_epu64(__X, __Y, __Z),
(__v2di)_mm_setzero_si128());
}

static __inline__ __m256i __DEFAULT_FN_ATTRS256
_mm256_mask_madd52hi_epu64 (__m256i __W, __mmask8 __M, __m256i __X, __m256i __Y)
{
return (__m256i)__builtin_ia32_selectq_256(__M,
(__v4di)_mm256_madd52hi_epu64(__W, __X, __Y),
(__v4di)__W);
static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_mask_madd52hi_epu64(
__m256i __W, __mmask8 __M, __m256i __X, __m256i __Y) {
return (__m256i)__builtin_ia32_selectq_256(
__M, (__v4di)_mm256_madd52hi_epu64(__W, __X, __Y), (__v4di)__W);
}

static __inline__ __m256i __DEFAULT_FN_ATTRS256
_mm256_maskz_madd52hi_epu64 (__mmask8 __M, __m256i __X, __m256i __Y, __m256i __Z)
{
return (__m256i)__builtin_ia32_selectq_256(__M,
(__v4di)_mm256_madd52hi_epu64(__X, __Y, __Z),
(__v4di)_mm256_setzero_si256());
static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_maskz_madd52hi_epu64(
__mmask8 __M, __m256i __X, __m256i __Y, __m256i __Z) {
return (__m256i)__builtin_ia32_selectq_256(
__M, (__v4di)_mm256_madd52hi_epu64(__X, __Y, __Z),
(__v4di)_mm256_setzero_si256());
}

static __inline__ __m128i __DEFAULT_FN_ATTRS128
_mm_mask_madd52lo_epu64 (__m128i __W, __mmask8 __M, __m128i __X, __m128i __Y)
{
return (__m128i)__builtin_ia32_selectq_128(__M,
(__v2di)_mm_madd52lo_epu64(__W, __X, __Y),
(__v2di)__W);
_mm_mask_madd52lo_epu64(__m128i __W, __mmask8 __M, __m128i __X, __m128i __Y) {
return (__m128i)__builtin_ia32_selectq_128(
__M, (__v2di)_mm_madd52lo_epu64(__W, __X, __Y), (__v2di)__W);
}

static __inline__ __m128i __DEFAULT_FN_ATTRS128
_mm_maskz_madd52lo_epu64 (__mmask8 __M, __m128i __X, __m128i __Y, __m128i __Z)
{
return (__m128i)__builtin_ia32_selectq_128(__M,
(__v2di)_mm_madd52lo_epu64(__X, __Y, __Z),
(__v2di)_mm_setzero_si128());
_mm_maskz_madd52lo_epu64(__mmask8 __M, __m128i __X, __m128i __Y, __m128i __Z) {
return (__m128i)__builtin_ia32_selectq_128(
__M, (__v2di)_mm_madd52lo_epu64(__X, __Y, __Z),
(__v2di)_mm_setzero_si128());
}

static __inline__ __m256i __DEFAULT_FN_ATTRS256
_mm256_mask_madd52lo_epu64 (__m256i __W, __mmask8 __M, __m256i __X, __m256i __Y)
{
return (__m256i)__builtin_ia32_selectq_256(__M,
(__v4di)_mm256_madd52lo_epu64(__W, __X, __Y),
(__v4di)__W);
static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_mask_madd52lo_epu64(
__m256i __W, __mmask8 __M, __m256i __X, __m256i __Y) {
return (__m256i)__builtin_ia32_selectq_256(
__M, (__v4di)_mm256_madd52lo_epu64(__W, __X, __Y), (__v4di)__W);
}

static __inline__ __m256i __DEFAULT_FN_ATTRS256
_mm256_maskz_madd52lo_epu64 (__mmask8 __M, __m256i __X, __m256i __Y, __m256i __Z)
{
return (__m256i)__builtin_ia32_selectq_256(__M,
(__v4di)_mm256_madd52lo_epu64(__X, __Y, __Z),
(__v4di)_mm256_setzero_si256());
static __inline__ __m256i __DEFAULT_FN_ATTRS256 _mm256_maskz_madd52lo_epu64(
__mmask8 __M, __m256i __X, __m256i __Y, __m256i __Z) {
return (__m256i)__builtin_ia32_selectq_256(
__M, (__v4di)_mm256_madd52lo_epu64(__X, __Y, __Z),
(__v4di)_mm256_setzero_si256());
}


#undef __DEFAULT_FN_ATTRS128
#undef __DEFAULT_FN_ATTRS256

Expand Down
9 changes: 9 additions & 0 deletions clang/lib/Headers/avxifmaintrin.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,21 @@
#define __AVXIFMAINTRIN_H

/* Define the default attributes for the functions in this file. */
#if defined(__cplusplus) && (__cplusplus >= 201103L)
#define __DEFAULT_FN_ATTRS128 \
constexpr __attribute__((__always_inline__, __nodebug__, \
__target__("avxifma"), __min_vector_width__(128)))
#define __DEFAULT_FN_ATTRS256 \
constexpr __attribute__((__always_inline__, __nodebug__, \
__target__("avxifma"), __min_vector_width__(256)))
#else
#define __DEFAULT_FN_ATTRS128 \
__attribute__((__always_inline__, __nodebug__, __target__("avxifma"), \
__min_vector_width__(128)))
#define __DEFAULT_FN_ATTRS256 \
__attribute__((__always_inline__, __nodebug__, __target__("avxifma"), \
__min_vector_width__(256)))
#endif

// must vex-encoding

Expand Down
Loading
Loading