diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td index d65b3a5d2f447..24810292d1d55 100644 --- a/clang/include/clang/Basic/Builtins.td +++ b/clang/include/clang/Basic/Builtins.td @@ -2588,6 +2588,24 @@ def InterlockedBittestAndReset_rel : MSLangBuiltin { let Prototype = "unsigned char(msint32_t volatile*, msint32_t)"; } +def InterlockedBittestAndReset64_acq : MSLangBuiltin { + let Spellings = ["_interlockedbittestandreset64_acq"]; + let Attributes = [NoThrow]; + let Prototype = "unsigned char(int64_t volatile*, int64_t)"; +} + +def InterlockedBittestAndReset64_nf : MSLangBuiltin { + let Spellings = ["_interlockedbittestandreset64_nf"]; + let Attributes = [NoThrow]; + let Prototype = "unsigned char(int64_t volatile*, int64_t)"; +} + +def InterlockedBittestAndReset64_rel : MSLangBuiltin { + let Spellings = ["_interlockedbittestandreset64_rel"]; + let Attributes = [NoThrow]; + let Prototype = "unsigned char(int64_t volatile*, int64_t)"; +} + def InterlockedBittestAndSet : MSLangBuiltin, MSInt32_64Template { let Spellings = ["_interlockedbittestandset"]; let Attributes = [NoThrow]; @@ -2612,6 +2630,24 @@ def InterlockedBittestAndSet_rel : MSLangBuiltin { let Prototype = "unsigned char(msint32_t volatile*, msint32_t)"; } +def InterlockedBittestAndSet64_acq : MSLangBuiltin { + let Spellings = ["_interlockedbittestandset64_acq"]; + let Attributes = [NoThrow]; + let Prototype = "unsigned char(int64_t volatile*, int64_t)"; +} + +def InterlockedBittestAndSet64_nf : MSLangBuiltin { + let Spellings = ["_interlockedbittestandset64_nf"]; + let Attributes = [NoThrow]; + let Prototype = "unsigned char(int64_t volatile*, int64_t)"; +} + +def InterlockedBittestAndSet64_rel : MSLangBuiltin { + let Spellings = ["_interlockedbittestandset64_rel"]; + let Attributes = [NoThrow]; + let Prototype = "unsigned char(int64_t volatile*, int64_t)"; +} + def IsoVolatileLoad : MSLangBuiltin, Int8_16_32_64Template { let Spellings = ["__iso_volatile_load"]; let Attributes = [NoThrow]; diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index 2a8722221f24b..48c91eb4a5b4f 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -1589,7 +1589,7 @@ BitTest BitTest::decodeBitTestBuiltin(unsigned BuiltinID) { case Builtin::BI_interlockedbittestandset: return {Set, Sequential, false}; - // X86-specific 64-bit variants. + // 64-bit variants. case Builtin::BI_bittest64: return {TestOnly, Unlocked, true}; case Builtin::BI_bittestandcomplement64: @@ -1616,6 +1616,18 @@ BitTest BitTest::decodeBitTestBuiltin(unsigned BuiltinID) { return {Reset, Release, false}; case Builtin::BI_interlockedbittestandreset_nf: return {Reset, NoFence, false}; + case Builtin::BI_interlockedbittestandreset64_acq: + return {Reset, Acquire, false}; + case Builtin::BI_interlockedbittestandreset64_rel: + return {Reset, Release, false}; + case Builtin::BI_interlockedbittestandreset64_nf: + return {Reset, NoFence, false}; + case Builtin::BI_interlockedbittestandset64_acq: + return {Set, Acquire, false}; + case Builtin::BI_interlockedbittestandset64_rel: + return {Set, Release, false}; + case Builtin::BI_interlockedbittestandset64_nf: + return {Set, NoFence, false}; } llvm_unreachable("expected only bittest intrinsics"); } @@ -5538,7 +5550,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID, case Builtin::BI_bittestandset: case Builtin::BI_interlockedbittestandreset: case Builtin::BI_interlockedbittestandreset64: + case Builtin::BI_interlockedbittestandreset64_acq: + case Builtin::BI_interlockedbittestandreset64_rel: + case Builtin::BI_interlockedbittestandreset64_nf: case Builtin::BI_interlockedbittestandset64: + case Builtin::BI_interlockedbittestandset64_acq: + case Builtin::BI_interlockedbittestandset64_rel: + case Builtin::BI_interlockedbittestandset64_nf: case Builtin::BI_interlockedbittestandset: case Builtin::BI_interlockedbittestandset_acq: case Builtin::BI_interlockedbittestandset_rel: diff --git a/clang/lib/Headers/intrin.h b/clang/lib/Headers/intrin.h index 39ccc97540b1e..d9382440f4b42 100644 --- a/clang/lib/Headers/intrin.h +++ b/clang/lib/Headers/intrin.h @@ -370,6 +370,18 @@ static __inline__ void __DEFAULT_FN_ATTRS __nop(void) { \*----------------------------------------------------------------------------*/ #if defined(__aarch64__) || defined(__arm64ec__) unsigned __int64 __getReg(int); +unsigned char _interlockedbittestandreset_acq(long volatile *, long); +unsigned char _interlockedbittestandreset_nf(long volatile *, long); +unsigned char _interlockedbittestandreset_rel(long volatile *, long); +unsigned char _interlockedbittestandreset64_acq(__int64 volatile *, __int64); +unsigned char _interlockedbittestandreset64_nf(__int64 volatile *, __int64); +unsigned char _interlockedbittestandreset64_rel(__int64 volatile *, __int64); +unsigned char _interlockedbittestandset_acq(long volatile *, long); +unsigned char _interlockedbittestandset_nf(long volatile *, long); +unsigned char _interlockedbittestandset_rel(long volatile *, long); +unsigned char _interlockedbittestandset64_acq(__int64 volatile *, __int64); +unsigned char _interlockedbittestandset64_nf(__int64 volatile *, __int64); +unsigned char _interlockedbittestandset64_rel(__int64 volatile *, __int64); long _InterlockedAdd(long volatile *, long); long _InterlockedAdd_acq(long volatile *, long); long _InterlockedAdd_nf(long volatile *, long); diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index 048b0892f6a31..926adebf7de0f 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -2364,6 +2364,17 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID, return ExprError(); break; + // The 64-bit acquire, release, and no fence variants are AArch64 only. + case Builtin::BI_interlockedbittestandreset64_acq: + case Builtin::BI_interlockedbittestandreset64_rel: + case Builtin::BI_interlockedbittestandreset64_nf: + case Builtin::BI_interlockedbittestandset64_acq: + case Builtin::BI_interlockedbittestandset64_rel: + case Builtin::BI_interlockedbittestandset64_nf: + if (CheckBuiltinTargetInSupported(*this, TheCall, {llvm::Triple::aarch64})) + return ExprError(); + break; + case Builtin::BI__builtin_set_flt_rounds: if (CheckBuiltinTargetInSupported( *this, TheCall, diff --git a/clang/test/CodeGen/bittest-intrin.c b/clang/test/CodeGen/bittest-intrin.c index 5641b04cf86e7..bc5ec67734501 100644 --- a/clang/test/CodeGen/bittest-intrin.c +++ b/clang/test/CodeGen/bittest-intrin.c @@ -1,6 +1,6 @@ // RUN: %clang_cc1 -fms-extensions -triple x86_64-windows-msvc %s -emit-llvm -o - | FileCheck %s --check-prefix=X64 // RUN: %clang_cc1 -fms-extensions -triple thumbv7-windows-msvc %s -emit-llvm -o - | FileCheck %s --check-prefix=ARM -// RUN: %clang_cc1 -fms-extensions -triple aarch64-windows-msvc %s -emit-llvm -o - | FileCheck %s --check-prefix=ARM +// RUN: %clang_cc1 -fms-extensions -triple aarch64-windows-msvc %s -emit-llvm -o - | FileCheck %s --check-prefix=ARM64 -check-prefix=ARM volatile unsigned char sink = 0; void test32(long *base, long idx) { @@ -10,7 +10,6 @@ void test32(long *base, long idx) { sink = _bittestandset(base, idx); sink = _interlockedbittestandreset(base, idx); sink = _interlockedbittestandset(base, idx); - sink = _interlockedbittestandset(base, idx); } void test64(__int64 *base, __int64 idx) { @@ -33,6 +32,17 @@ void test_arm(long *base, long idx) { } #endif +#if defined(_M_ARM64) +void test_arm64(__int64 *base, __int64 idx) { + sink = _interlockedbittestandreset64_acq(base, idx); + sink = _interlockedbittestandreset64_rel(base, idx); + sink = _interlockedbittestandreset64_nf(base, idx); + sink = _interlockedbittestandset64_acq(base, idx); + sink = _interlockedbittestandset64_rel(base, idx); + sink = _interlockedbittestandset64_nf(base, idx); +} +#endif + // X64-LABEL: define dso_local void @test32(ptr noundef %base, i32 noundef %idx) // X64: call i8 asm sideeffect "btl $2, ($1)", "={@ccc},r,r,~{cc},~{memory},~{dirflag},~{fpsr},~{flags}"(ptr %{{.*}}, i32 {{.*}}) // X64: call i8 asm sideeffect "btcl $2, ($1)", "={@ccc},r,r,~{cc},~{memory},~{dirflag},~{fpsr},~{flags}"(ptr %{{.*}}, i32 {{.*}}) @@ -117,13 +127,122 @@ void test_arm(long *base, long idx) { // ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1 // ARM: store volatile i8 %[[RES]], ptr @sink, align 1 +// ARM-LABEL: define dso_local {{.*}}void @test64(ptr noundef %base, i64 noundef %idx) +// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3 +// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]] +// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8 +// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7 +// ARM: %[[BYTE:[^ ]*]] = load i8, ptr %[[BYTEADDR]], align 1 +// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]] +// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1 +// ARM: store volatile i8 %[[RES]], ptr @sink, align 1 -// Just look for the atomicrmw instructions. +// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3 +// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]] +// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8 +// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7 +// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]] +// ARM: %[[BYTE:[^ ]*]] = load i8, ptr %[[BYTEADDR]], align 1 +// ARM: %[[NEWBYTE:[^ ]*]] = xor i8 %[[BYTE]], %[[MASK]] +// ARM: store i8 %[[NEWBYTE]], ptr %[[BYTEADDR]], align 1 +// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]] +// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1 +// ARM: store volatile i8 %[[RES]], ptr @sink, align 1 + +// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3 +// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]] +// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8 +// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7 +// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]] +// ARM: %[[BYTE:[^ ]*]] = load i8, ptr %[[BYTEADDR]], align 1 +// ARM: %[[NOTMASK:[^ ]*]] = xor i8 %[[MASK]], -1 +// ARM: %[[NEWBYTE:[^ ]*]] = and i8 %[[BYTE]], %[[NOTMASK]] +// ARM: store i8 %[[NEWBYTE]], ptr %[[BYTEADDR]], align 1 +// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]] +// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1 +// ARM: store volatile i8 %[[RES]], ptr @sink, align 1 + +// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3 +// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]] +// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8 +// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7 +// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]] +// ARM: %[[BYTE:[^ ]*]] = load i8, ptr %[[BYTEADDR]], align 1 +// ARM: %[[NEWBYTE:[^ ]*]] = or i8 %[[BYTE]], %[[MASK]] +// ARM: store i8 %[[NEWBYTE]], ptr %[[BYTEADDR]], align 1 +// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]] +// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1 +// ARM: store volatile i8 %[[RES]], ptr @sink, align 1 + +// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3 +// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]] +// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8 +// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7 +// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]] +// ARM: %[[NOTMASK:[^ ]*]] = xor i8 %[[MASK]], -1 +// ARM: %[[BYTE:[^ ]*]] = atomicrmw and ptr %[[BYTEADDR]], i8 %[[NOTMASK]] seq_cst, align 1 +// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]] +// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1 +// ARM: store volatile i8 %[[RES]], ptr @sink, align 1 + +// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3 +// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]] +// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8 +// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7 +// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]] +// ARM: %[[BYTE:[^ ]*]] = atomicrmw or ptr %[[BYTEADDR]], i8 %[[MASK]] seq_cst, align 1 +// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]] +// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1 +// ARM: store volatile i8 %[[RES]], ptr @sink, align 1 // ARM-LABEL: define dso_local {{.*}}void @test_arm(ptr noundef %base, i32 noundef %idx) -// ARM: atomicrmw and ptr %{{.*}}, i8 {{.*}} acquire, align 1 +// ARM: %[[IDXHI:[^ ]*]] = ashr i32 %{{.*}}, 3 +// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i32 %[[IDXHI]] +// ARM: %[[IDX8:[^ ]*]] = trunc i32 %{{.*}} to i8 +// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7 +// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]] +// ARM: %[[NOTMASK:[^ ]*]] = xor i8 %[[MASK]], -1 +// ARM: %[[BYTE:[^ ]*]] = atomicrmw and ptr %[[BYTEADDR]], i8 %[[NOTMASK]] acquire, align 1 +// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]] +// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1 +// ARM: store volatile i8 %[[RES]], ptr @sink, align 1 +// Just look for the atomicrmw instructions. // ARM: atomicrmw and ptr %{{.*}}, i8 {{.*}} release, align 1 // ARM: atomicrmw and ptr %{{.*}}, i8 {{.*}} monotonic, align 1 -// ARM: atomicrmw or ptr %{{.*}}, i8 {{.*}} acquire, align 1 +// ARM: %[[IDXHI:[^ ]*]] = ashr i32 %{{.*}}, 3 +// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i32 %[[IDXHI]] +// ARM: %[[IDX8:[^ ]*]] = trunc i32 %{{.*}} to i8 +// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7 +// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]] +// ARM: %[[BYTE:[^ ]*]] = atomicrmw or ptr %[[BYTEADDR]], i8 %[[MASK]] acquire, align 1 +// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]] +// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1 +// ARM: store volatile i8 %[[RES]], ptr @sink, align 1 +// Just look for the atomicrmw instructions. // ARM: atomicrmw or ptr %{{.*}}, i8 {{.*}} release, align 1 // ARM: atomicrmw or ptr %{{.*}}, i8 {{.*}} monotonic, align 1 + +// ARM64-LABEL: define dso_local void @test_arm64(ptr noundef %base, i64 noundef %idx) +// ARM64: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3 +// ARM64: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]] +// ARM64: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8 +// ARM64: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7 +// ARM64: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]] +// ARM64: %[[NOTMASK:[^ ]*]] = xor i8 %[[MASK]], -1 +// ARM64: %[[BYTE:[^ ]*]] = atomicrmw and ptr %[[BYTEADDR]], i8 %[[NOTMASK]] acquire, align 1 +// ARM64: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]] +// ARM64: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1 +// ARM64: store volatile i8 %[[RES]], ptr @sink, align 1 +// ARM64: atomicrmw and ptr %{{.*}}, i8 {{.*}} release, align 1 +// ARM64: atomicrmw and ptr %{{.*}}, i8 {{.*}} monotonic, align 1 +// ARM64: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3 +// ARM64: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]] +// ARM64: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8 +// ARM64: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7 +// ARM64: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]] +// ARM64: %[[BYTE:[^ ]*]] = atomicrmw or ptr %[[BYTEADDR]], i8 %[[MASK]] acquire, align 1 +// ARM64: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]] +// ARM64: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1 +// ARM64: store volatile i8 %[[RES]], ptr @sink, align 1 +// ARM64: atomicrmw or ptr %{{.*}}, i8 {{.*}} release, align 1 +// ARM64: atomicrmw or ptr %{{.*}}, i8 {{.*}} monotonic, align 1