[Clang][AArch64] _interlockedbittestand{set,reset}64_{acq,rel,nf} support for AArch64 #145980
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
llvmbot
added
clang
Member
|
@llvm/pr-subscribers-backend-aarch64 @llvm/pr-subscribers-clang-codegen Author: Adam Glass (AdamGlass) ChangesAdds interlockedbittestand{set,reset}64{acq,rel,nf} support for AArch64 Includes test changes and has been clang-format-ed [email protected] @dpaoliello Full diff: https://github.com/llvm/llvm-project/pull/145980.diff 5 Files Affected:
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index d65b3a5d2f447..24810292d1d55 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -2588,6 +2588,24 @@ def InterlockedBittestAndReset_rel : MSLangBuiltin {
let Prototype = "unsigned char(msint32_t volatile*, msint32_t)";
}
+def InterlockedBittestAndReset64_acq : MSLangBuiltin {
+ let Spellings = ["_interlockedbittestandreset64_acq"];
+ let Attributes = [NoThrow];
+ let Prototype = "unsigned char(int64_t volatile*, int64_t)";
+}
+
+def InterlockedBittestAndReset64_nf : MSLangBuiltin {
+ let Spellings = ["_interlockedbittestandreset64_nf"];
+ let Attributes = [NoThrow];
+ let Prototype = "unsigned char(int64_t volatile*, int64_t)";
+}
+
+def InterlockedBittestAndReset64_rel : MSLangBuiltin {
+ let Spellings = ["_interlockedbittestandreset64_rel"];
+ let Attributes = [NoThrow];
+ let Prototype = "unsigned char(int64_t volatile*, int64_t)";
+}
+
def InterlockedBittestAndSet : MSLangBuiltin, MSInt32_64Template {
let Spellings = ["_interlockedbittestandset"];
let Attributes = [NoThrow];
@@ -2612,6 +2630,24 @@ def InterlockedBittestAndSet_rel : MSLangBuiltin {
let Prototype = "unsigned char(msint32_t volatile*, msint32_t)";
}
+def InterlockedBittestAndSet64_acq : MSLangBuiltin {
+ let Spellings = ["_interlockedbittestandset64_acq"];
+ let Attributes = [NoThrow];
+ let Prototype = "unsigned char(int64_t volatile*, int64_t)";
+}
+
+def InterlockedBittestAndSet64_nf : MSLangBuiltin {
+ let Spellings = ["_interlockedbittestandset64_nf"];
+ let Attributes = [NoThrow];
+ let Prototype = "unsigned char(int64_t volatile*, int64_t)";
+}
+
+def InterlockedBittestAndSet64_rel : MSLangBuiltin {
+ let Spellings = ["_interlockedbittestandset64_rel"];
+ let Attributes = [NoThrow];
+ let Prototype = "unsigned char(int64_t volatile*, int64_t)";
+}
+
def IsoVolatileLoad : MSLangBuiltin, Int8_16_32_64Template {
let Spellings = ["__iso_volatile_load"];
let Attributes = [NoThrow];
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 2a8722221f24b..48c91eb4a5b4f 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -1589,7 +1589,7 @@ BitTest BitTest::decodeBitTestBuiltin(unsigned BuiltinID) {
case Builtin::BI_interlockedbittestandset:
return {Set, Sequential, false};
- // X86-specific 64-bit variants.
+ // 64-bit variants.
case Builtin::BI_bittest64:
return {TestOnly, Unlocked, true};
case Builtin::BI_bittestandcomplement64:
@@ -1616,6 +1616,18 @@ BitTest BitTest::decodeBitTestBuiltin(unsigned BuiltinID) {
return {Reset, Release, false};
case Builtin::BI_interlockedbittestandreset_nf:
return {Reset, NoFence, false};
+ case Builtin::BI_interlockedbittestandreset64_acq:
+ return {Reset, Acquire, false};
+ case Builtin::BI_interlockedbittestandreset64_rel:
+ return {Reset, Release, false};
+ case Builtin::BI_interlockedbittestandreset64_nf:
+ return {Reset, NoFence, false};
+ case Builtin::BI_interlockedbittestandset64_acq:
+ return {Set, Acquire, false};
+ case Builtin::BI_interlockedbittestandset64_rel:
+ return {Set, Release, false};
+ case Builtin::BI_interlockedbittestandset64_nf:
+ return {Set, NoFence, false};
}
llvm_unreachable("expected only bittest intrinsics");
}
@@ -5538,7 +5550,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
case Builtin::BI_bittestandset:
case Builtin::BI_interlockedbittestandreset:
case Builtin::BI_interlockedbittestandreset64:
+ case Builtin::BI_interlockedbittestandreset64_acq:
+ case Builtin::BI_interlockedbittestandreset64_rel:
+ case Builtin::BI_interlockedbittestandreset64_nf:
case Builtin::BI_interlockedbittestandset64:
+ case Builtin::BI_interlockedbittestandset64_acq:
+ case Builtin::BI_interlockedbittestandset64_rel:
+ case Builtin::BI_interlockedbittestandset64_nf:
case Builtin::BI_interlockedbittestandset:
case Builtin::BI_interlockedbittestandset_acq:
case Builtin::BI_interlockedbittestandset_rel:
diff --git a/clang/lib/Headers/intrin.h b/clang/lib/Headers/intrin.h
index 39ccc97540b1e..d9382440f4b42 100644
--- a/clang/lib/Headers/intrin.h
+++ b/clang/lib/Headers/intrin.h
@@ -370,6 +370,18 @@ static __inline__ void __DEFAULT_FN_ATTRS __nop(void) {
\*----------------------------------------------------------------------------*/
#if defined(__aarch64__) || defined(__arm64ec__)
unsigned __int64 __getReg(int);
+unsigned char _interlockedbittestandreset_acq(long volatile *, long);
+unsigned char _interlockedbittestandreset_nf(long volatile *, long);
+unsigned char _interlockedbittestandreset_rel(long volatile *, long);
+unsigned char _interlockedbittestandreset64_acq(__int64 volatile *, __int64);
+unsigned char _interlockedbittestandreset64_nf(__int64 volatile *, __int64);
+unsigned char _interlockedbittestandreset64_rel(__int64 volatile *, __int64);
+unsigned char _interlockedbittestandset_acq(long volatile *, long);
+unsigned char _interlockedbittestandset_nf(long volatile *, long);
+unsigned char _interlockedbittestandset_rel(long volatile *, long);
+unsigned char _interlockedbittestandset64_acq(__int64 volatile *, __int64);
+unsigned char _interlockedbittestandset64_nf(__int64 volatile *, __int64);
+unsigned char _interlockedbittestandset64_rel(__int64 volatile *, __int64);
long _InterlockedAdd(long volatile *, long);
long _InterlockedAdd_acq(long volatile *, long);
long _InterlockedAdd_nf(long volatile *, long);
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 048b0892f6a31..926adebf7de0f 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -2364,6 +2364,17 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
return ExprError();
break;
+ // The 64-bit acquire, release, and no fence variants are AArch64 only.
+ case Builtin::BI_interlockedbittestandreset64_acq:
+ case Builtin::BI_interlockedbittestandreset64_rel:
+ case Builtin::BI_interlockedbittestandreset64_nf:
+ case Builtin::BI_interlockedbittestandset64_acq:
+ case Builtin::BI_interlockedbittestandset64_rel:
+ case Builtin::BI_interlockedbittestandset64_nf:
+ if (CheckBuiltinTargetInSupported(*this, TheCall, {llvm::Triple::aarch64}))
+ return ExprError();
+ break;
+
case Builtin::BI__builtin_set_flt_rounds:
if (CheckBuiltinTargetInSupported(
*this, TheCall,
diff --git a/clang/test/CodeGen/bittest-intrin.c b/clang/test/CodeGen/bittest-intrin.c
index 5641b04cf86e7..bc5ec67734501 100644
--- a/clang/test/CodeGen/bittest-intrin.c
+++ b/clang/test/CodeGen/bittest-intrin.c
@@ -1,6 +1,6 @@
// RUN: %clang_cc1 -fms-extensions -triple x86_64-windows-msvc %s -emit-llvm -o - | FileCheck %s --check-prefix=X64
// RUN: %clang_cc1 -fms-extensions -triple thumbv7-windows-msvc %s -emit-llvm -o - | FileCheck %s --check-prefix=ARM
-// RUN: %clang_cc1 -fms-extensions -triple aarch64-windows-msvc %s -emit-llvm -o - | FileCheck %s --check-prefix=ARM
+// RUN: %clang_cc1 -fms-extensions -triple aarch64-windows-msvc %s -emit-llvm -o - | FileCheck %s --check-prefix=ARM64 -check-prefix=ARM
volatile unsigned char sink = 0;
void test32(long *base, long idx) {
@@ -10,7 +10,6 @@ void test32(long *base, long idx) {
sink = _bittestandset(base, idx);
sink = _interlockedbittestandreset(base, idx);
sink = _interlockedbittestandset(base, idx);
- sink = _interlockedbittestandset(base, idx);
}
void test64(__int64 *base, __int64 idx) {
@@ -33,6 +32,17 @@ void test_arm(long *base, long idx) {
}
#endif
+#if defined(_M_ARM64)
+void test_arm64(__int64 *base, __int64 idx) {
+ sink = _interlockedbittestandreset64_acq(base, idx);
+ sink = _interlockedbittestandreset64_rel(base, idx);
+ sink = _interlockedbittestandreset64_nf(base, idx);
+ sink = _interlockedbittestandset64_acq(base, idx);
+ sink = _interlockedbittestandset64_rel(base, idx);
+ sink = _interlockedbittestandset64_nf(base, idx);
+}
+#endif
+
// X64-LABEL: define dso_local void @test32(ptr noundef %base, i32 noundef %idx)
// X64: call i8 asm sideeffect "btl $2, ($1)", "={@ccc},r,r,~{cc},~{memory},~{dirflag},~{fpsr},~{flags}"(ptr %{{.*}}, i32 {{.*}})
// X64: call i8 asm sideeffect "btcl $2, ($1)", "={@ccc},r,r,~{cc},~{memory},~{dirflag},~{fpsr},~{flags}"(ptr %{{.*}}, i32 {{.*}})
@@ -117,13 +127,122 @@ void test_arm(long *base, long idx) {
// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
+// ARM-LABEL: define dso_local {{.*}}void @test64(ptr noundef %base, i64 noundef %idx)
+// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
+// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
+// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
+// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM: %[[BYTE:[^ ]*]] = load i8, ptr %[[BYTEADDR]], align 1
+// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
-// Just look for the atomicrmw instructions.
+// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
+// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
+// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
+// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
+// ARM: %[[BYTE:[^ ]*]] = load i8, ptr %[[BYTEADDR]], align 1
+// ARM: %[[NEWBYTE:[^ ]*]] = xor i8 %[[BYTE]], %[[MASK]]
+// ARM: store i8 %[[NEWBYTE]], ptr %[[BYTEADDR]], align 1
+// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
+
+// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
+// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
+// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
+// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
+// ARM: %[[BYTE:[^ ]*]] = load i8, ptr %[[BYTEADDR]], align 1
+// ARM: %[[NOTMASK:[^ ]*]] = xor i8 %[[MASK]], -1
+// ARM: %[[NEWBYTE:[^ ]*]] = and i8 %[[BYTE]], %[[NOTMASK]]
+// ARM: store i8 %[[NEWBYTE]], ptr %[[BYTEADDR]], align 1
+// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
+
+// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
+// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
+// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
+// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
+// ARM: %[[BYTE:[^ ]*]] = load i8, ptr %[[BYTEADDR]], align 1
+// ARM: %[[NEWBYTE:[^ ]*]] = or i8 %[[BYTE]], %[[MASK]]
+// ARM: store i8 %[[NEWBYTE]], ptr %[[BYTEADDR]], align 1
+// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
+
+// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
+// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
+// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
+// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
+// ARM: %[[NOTMASK:[^ ]*]] = xor i8 %[[MASK]], -1
+// ARM: %[[BYTE:[^ ]*]] = atomicrmw and ptr %[[BYTEADDR]], i8 %[[NOTMASK]] seq_cst, align 1
+// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
+
+// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
+// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
+// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
+// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
+// ARM: %[[BYTE:[^ ]*]] = atomicrmw or ptr %[[BYTEADDR]], i8 %[[MASK]] seq_cst, align 1
+// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
// ARM-LABEL: define dso_local {{.*}}void @test_arm(ptr noundef %base, i32 noundef %idx)
-// ARM: atomicrmw and ptr %{{.*}}, i8 {{.*}} acquire, align 1
+// ARM: %[[IDXHI:[^ ]*]] = ashr i32 %{{.*}}, 3
+// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i32 %[[IDXHI]]
+// ARM: %[[IDX8:[^ ]*]] = trunc i32 %{{.*}} to i8
+// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
+// ARM: %[[NOTMASK:[^ ]*]] = xor i8 %[[MASK]], -1
+// ARM: %[[BYTE:[^ ]*]] = atomicrmw and ptr %[[BYTEADDR]], i8 %[[NOTMASK]] acquire, align 1
+// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
+// Just look for the atomicrmw instructions.
// ARM: atomicrmw and ptr %{{.*}}, i8 {{.*}} release, align 1
// ARM: atomicrmw and ptr %{{.*}}, i8 {{.*}} monotonic, align 1
-// ARM: atomicrmw or ptr %{{.*}}, i8 {{.*}} acquire, align 1
+// ARM: %[[IDXHI:[^ ]*]] = ashr i32 %{{.*}}, 3
+// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i32 %[[IDXHI]]
+// ARM: %[[IDX8:[^ ]*]] = trunc i32 %{{.*}} to i8
+// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
+// ARM: %[[BYTE:[^ ]*]] = atomicrmw or ptr %[[BYTEADDR]], i8 %[[MASK]] acquire, align 1
+// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
+// Just look for the atomicrmw instructions.
// ARM: atomicrmw or ptr %{{.*}}, i8 {{.*}} release, align 1
// ARM: atomicrmw or ptr %{{.*}}, i8 {{.*}} monotonic, align 1
+
+// ARM64-LABEL: define dso_local void @test_arm64(ptr noundef %base, i64 noundef %idx)
+// ARM64: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
+// ARM64: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
+// ARM64: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
+// ARM64: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM64: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
+// ARM64: %[[NOTMASK:[^ ]*]] = xor i8 %[[MASK]], -1
+// ARM64: %[[BYTE:[^ ]*]] = atomicrmw and ptr %[[BYTEADDR]], i8 %[[NOTMASK]] acquire, align 1
+// ARM64: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM64: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM64: store volatile i8 %[[RES]], ptr @sink, align 1
+// ARM64: atomicrmw and ptr %{{.*}}, i8 {{.*}} release, align 1
+// ARM64: atomicrmw and ptr %{{.*}}, i8 {{.*}} monotonic, align 1
+// ARM64: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
+// ARM64: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
+// ARM64: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
+// ARM64: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM64: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
+// ARM64: %[[BYTE:[^ ]*]] = atomicrmw or ptr %[[BYTEADDR]], i8 %[[MASK]] acquire, align 1
+// ARM64: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM64: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM64: store volatile i8 %[[RES]], ptr @sink, align 1
+// ARM64: atomicrmw or ptr %{{.*}}, i8 {{.*}} release, align 1
+// ARM64: atomicrmw or ptr %{{.*}}, i8 {{.*}} monotonic, align 1
|
Member
|
@llvm/pr-subscribers-clang Author: Adam Glass (AdamGlass) ChangesAdds interlockedbittestand{set,reset}64{acq,rel,nf} support for AArch64 Includes test changes and has been clang-format-ed [email protected] @dpaoliello Full diff: https://github.com/llvm/llvm-project/pull/145980.diff 5 Files Affected:
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index d65b3a5d2f447..24810292d1d55 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -2588,6 +2588,24 @@ def InterlockedBittestAndReset_rel : MSLangBuiltin {
let Prototype = "unsigned char(msint32_t volatile*, msint32_t)";
}
+def InterlockedBittestAndReset64_acq : MSLangBuiltin {
+ let Spellings = ["_interlockedbittestandreset64_acq"];
+ let Attributes = [NoThrow];
+ let Prototype = "unsigned char(int64_t volatile*, int64_t)";
+}
+
+def InterlockedBittestAndReset64_nf : MSLangBuiltin {
+ let Spellings = ["_interlockedbittestandreset64_nf"];
+ let Attributes = [NoThrow];
+ let Prototype = "unsigned char(int64_t volatile*, int64_t)";
+}
+
+def InterlockedBittestAndReset64_rel : MSLangBuiltin {
+ let Spellings = ["_interlockedbittestandreset64_rel"];
+ let Attributes = [NoThrow];
+ let Prototype = "unsigned char(int64_t volatile*, int64_t)";
+}
+
def InterlockedBittestAndSet : MSLangBuiltin, MSInt32_64Template {
let Spellings = ["_interlockedbittestandset"];
let Attributes = [NoThrow];
@@ -2612,6 +2630,24 @@ def InterlockedBittestAndSet_rel : MSLangBuiltin {
let Prototype = "unsigned char(msint32_t volatile*, msint32_t)";
}
+def InterlockedBittestAndSet64_acq : MSLangBuiltin {
+ let Spellings = ["_interlockedbittestandset64_acq"];
+ let Attributes = [NoThrow];
+ let Prototype = "unsigned char(int64_t volatile*, int64_t)";
+}
+
+def InterlockedBittestAndSet64_nf : MSLangBuiltin {
+ let Spellings = ["_interlockedbittestandset64_nf"];
+ let Attributes = [NoThrow];
+ let Prototype = "unsigned char(int64_t volatile*, int64_t)";
+}
+
+def InterlockedBittestAndSet64_rel : MSLangBuiltin {
+ let Spellings = ["_interlockedbittestandset64_rel"];
+ let Attributes = [NoThrow];
+ let Prototype = "unsigned char(int64_t volatile*, int64_t)";
+}
+
def IsoVolatileLoad : MSLangBuiltin, Int8_16_32_64Template {
let Spellings = ["__iso_volatile_load"];
let Attributes = [NoThrow];
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 2a8722221f24b..48c91eb4a5b4f 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -1589,7 +1589,7 @@ BitTest BitTest::decodeBitTestBuiltin(unsigned BuiltinID) {
case Builtin::BI_interlockedbittestandset:
return {Set, Sequential, false};
- // X86-specific 64-bit variants.
+ // 64-bit variants.
case Builtin::BI_bittest64:
return {TestOnly, Unlocked, true};
case Builtin::BI_bittestandcomplement64:
@@ -1616,6 +1616,18 @@ BitTest BitTest::decodeBitTestBuiltin(unsigned BuiltinID) {
return {Reset, Release, false};
case Builtin::BI_interlockedbittestandreset_nf:
return {Reset, NoFence, false};
+ case Builtin::BI_interlockedbittestandreset64_acq:
+ return {Reset, Acquire, false};
+ case Builtin::BI_interlockedbittestandreset64_rel:
+ return {Reset, Release, false};
+ case Builtin::BI_interlockedbittestandreset64_nf:
+ return {Reset, NoFence, false};
+ case Builtin::BI_interlockedbittestandset64_acq:
+ return {Set, Acquire, false};
+ case Builtin::BI_interlockedbittestandset64_rel:
+ return {Set, Release, false};
+ case Builtin::BI_interlockedbittestandset64_nf:
+ return {Set, NoFence, false};
}
llvm_unreachable("expected only bittest intrinsics");
}
@@ -5538,7 +5550,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
case Builtin::BI_bittestandset:
case Builtin::BI_interlockedbittestandreset:
case Builtin::BI_interlockedbittestandreset64:
+ case Builtin::BI_interlockedbittestandreset64_acq:
+ case Builtin::BI_interlockedbittestandreset64_rel:
+ case Builtin::BI_interlockedbittestandreset64_nf:
case Builtin::BI_interlockedbittestandset64:
+ case Builtin::BI_interlockedbittestandset64_acq:
+ case Builtin::BI_interlockedbittestandset64_rel:
+ case Builtin::BI_interlockedbittestandset64_nf:
case Builtin::BI_interlockedbittestandset:
case Builtin::BI_interlockedbittestandset_acq:
case Builtin::BI_interlockedbittestandset_rel:
diff --git a/clang/lib/Headers/intrin.h b/clang/lib/Headers/intrin.h
index 39ccc97540b1e..d9382440f4b42 100644
--- a/clang/lib/Headers/intrin.h
+++ b/clang/lib/Headers/intrin.h
@@ -370,6 +370,18 @@ static __inline__ void __DEFAULT_FN_ATTRS __nop(void) {
\*----------------------------------------------------------------------------*/
#if defined(__aarch64__) || defined(__arm64ec__)
unsigned __int64 __getReg(int);
+unsigned char _interlockedbittestandreset_acq(long volatile *, long);
+unsigned char _interlockedbittestandreset_nf(long volatile *, long);
+unsigned char _interlockedbittestandreset_rel(long volatile *, long);
+unsigned char _interlockedbittestandreset64_acq(__int64 volatile *, __int64);
+unsigned char _interlockedbittestandreset64_nf(__int64 volatile *, __int64);
+unsigned char _interlockedbittestandreset64_rel(__int64 volatile *, __int64);
+unsigned char _interlockedbittestandset_acq(long volatile *, long);
+unsigned char _interlockedbittestandset_nf(long volatile *, long);
+unsigned char _interlockedbittestandset_rel(long volatile *, long);
+unsigned char _interlockedbittestandset64_acq(__int64 volatile *, __int64);
+unsigned char _interlockedbittestandset64_nf(__int64 volatile *, __int64);
+unsigned char _interlockedbittestandset64_rel(__int64 volatile *, __int64);
long _InterlockedAdd(long volatile *, long);
long _InterlockedAdd_acq(long volatile *, long);
long _InterlockedAdd_nf(long volatile *, long);
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index 048b0892f6a31..926adebf7de0f 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -2364,6 +2364,17 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
return ExprError();
break;
+ // The 64-bit acquire, release, and no fence variants are AArch64 only.
+ case Builtin::BI_interlockedbittestandreset64_acq:
+ case Builtin::BI_interlockedbittestandreset64_rel:
+ case Builtin::BI_interlockedbittestandreset64_nf:
+ case Builtin::BI_interlockedbittestandset64_acq:
+ case Builtin::BI_interlockedbittestandset64_rel:
+ case Builtin::BI_interlockedbittestandset64_nf:
+ if (CheckBuiltinTargetInSupported(*this, TheCall, {llvm::Triple::aarch64}))
+ return ExprError();
+ break;
+
case Builtin::BI__builtin_set_flt_rounds:
if (CheckBuiltinTargetInSupported(
*this, TheCall,
diff --git a/clang/test/CodeGen/bittest-intrin.c b/clang/test/CodeGen/bittest-intrin.c
index 5641b04cf86e7..bc5ec67734501 100644
--- a/clang/test/CodeGen/bittest-intrin.c
+++ b/clang/test/CodeGen/bittest-intrin.c
@@ -1,6 +1,6 @@
// RUN: %clang_cc1 -fms-extensions -triple x86_64-windows-msvc %s -emit-llvm -o - | FileCheck %s --check-prefix=X64
// RUN: %clang_cc1 -fms-extensions -triple thumbv7-windows-msvc %s -emit-llvm -o - | FileCheck %s --check-prefix=ARM
-// RUN: %clang_cc1 -fms-extensions -triple aarch64-windows-msvc %s -emit-llvm -o - | FileCheck %s --check-prefix=ARM
+// RUN: %clang_cc1 -fms-extensions -triple aarch64-windows-msvc %s -emit-llvm -o - | FileCheck %s --check-prefix=ARM64 -check-prefix=ARM
volatile unsigned char sink = 0;
void test32(long *base, long idx) {
@@ -10,7 +10,6 @@ void test32(long *base, long idx) {
sink = _bittestandset(base, idx);
sink = _interlockedbittestandreset(base, idx);
sink = _interlockedbittestandset(base, idx);
- sink = _interlockedbittestandset(base, idx);
}
void test64(__int64 *base, __int64 idx) {
@@ -33,6 +32,17 @@ void test_arm(long *base, long idx) {
}
#endif
+#if defined(_M_ARM64)
+void test_arm64(__int64 *base, __int64 idx) {
+ sink = _interlockedbittestandreset64_acq(base, idx);
+ sink = _interlockedbittestandreset64_rel(base, idx);
+ sink = _interlockedbittestandreset64_nf(base, idx);
+ sink = _interlockedbittestandset64_acq(base, idx);
+ sink = _interlockedbittestandset64_rel(base, idx);
+ sink = _interlockedbittestandset64_nf(base, idx);
+}
+#endif
+
// X64-LABEL: define dso_local void @test32(ptr noundef %base, i32 noundef %idx)
// X64: call i8 asm sideeffect "btl $2, ($1)", "={@ccc},r,r,~{cc},~{memory},~{dirflag},~{fpsr},~{flags}"(ptr %{{.*}}, i32 {{.*}})
// X64: call i8 asm sideeffect "btcl $2, ($1)", "={@ccc},r,r,~{cc},~{memory},~{dirflag},~{fpsr},~{flags}"(ptr %{{.*}}, i32 {{.*}})
@@ -117,13 +127,122 @@ void test_arm(long *base, long idx) {
// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
+// ARM-LABEL: define dso_local {{.*}}void @test64(ptr noundef %base, i64 noundef %idx)
+// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
+// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
+// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
+// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM: %[[BYTE:[^ ]*]] = load i8, ptr %[[BYTEADDR]], align 1
+// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
-// Just look for the atomicrmw instructions.
+// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
+// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
+// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
+// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
+// ARM: %[[BYTE:[^ ]*]] = load i8, ptr %[[BYTEADDR]], align 1
+// ARM: %[[NEWBYTE:[^ ]*]] = xor i8 %[[BYTE]], %[[MASK]]
+// ARM: store i8 %[[NEWBYTE]], ptr %[[BYTEADDR]], align 1
+// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
+
+// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
+// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
+// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
+// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
+// ARM: %[[BYTE:[^ ]*]] = load i8, ptr %[[BYTEADDR]], align 1
+// ARM: %[[NOTMASK:[^ ]*]] = xor i8 %[[MASK]], -1
+// ARM: %[[NEWBYTE:[^ ]*]] = and i8 %[[BYTE]], %[[NOTMASK]]
+// ARM: store i8 %[[NEWBYTE]], ptr %[[BYTEADDR]], align 1
+// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
+
+// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
+// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
+// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
+// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
+// ARM: %[[BYTE:[^ ]*]] = load i8, ptr %[[BYTEADDR]], align 1
+// ARM: %[[NEWBYTE:[^ ]*]] = or i8 %[[BYTE]], %[[MASK]]
+// ARM: store i8 %[[NEWBYTE]], ptr %[[BYTEADDR]], align 1
+// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
+
+// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
+// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
+// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
+// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
+// ARM: %[[NOTMASK:[^ ]*]] = xor i8 %[[MASK]], -1
+// ARM: %[[BYTE:[^ ]*]] = atomicrmw and ptr %[[BYTEADDR]], i8 %[[NOTMASK]] seq_cst, align 1
+// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
+
+// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
+// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
+// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
+// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
+// ARM: %[[BYTE:[^ ]*]] = atomicrmw or ptr %[[BYTEADDR]], i8 %[[MASK]] seq_cst, align 1
+// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
// ARM-LABEL: define dso_local {{.*}}void @test_arm(ptr noundef %base, i32 noundef %idx)
-// ARM: atomicrmw and ptr %{{.*}}, i8 {{.*}} acquire, align 1
+// ARM: %[[IDXHI:[^ ]*]] = ashr i32 %{{.*}}, 3
+// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i32 %[[IDXHI]]
+// ARM: %[[IDX8:[^ ]*]] = trunc i32 %{{.*}} to i8
+// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
+// ARM: %[[NOTMASK:[^ ]*]] = xor i8 %[[MASK]], -1
+// ARM: %[[BYTE:[^ ]*]] = atomicrmw and ptr %[[BYTEADDR]], i8 %[[NOTMASK]] acquire, align 1
+// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
+// Just look for the atomicrmw instructions.
// ARM: atomicrmw and ptr %{{.*}}, i8 {{.*}} release, align 1
// ARM: atomicrmw and ptr %{{.*}}, i8 {{.*}} monotonic, align 1
-// ARM: atomicrmw or ptr %{{.*}}, i8 {{.*}} acquire, align 1
+// ARM: %[[IDXHI:[^ ]*]] = ashr i32 %{{.*}}, 3
+// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i32 %[[IDXHI]]
+// ARM: %[[IDX8:[^ ]*]] = trunc i32 %{{.*}} to i8
+// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
+// ARM: %[[BYTE:[^ ]*]] = atomicrmw or ptr %[[BYTEADDR]], i8 %[[MASK]] acquire, align 1
+// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
+// Just look for the atomicrmw instructions.
// ARM: atomicrmw or ptr %{{.*}}, i8 {{.*}} release, align 1
// ARM: atomicrmw or ptr %{{.*}}, i8 {{.*}} monotonic, align 1
+
+// ARM64-LABEL: define dso_local void @test_arm64(ptr noundef %base, i64 noundef %idx)
+// ARM64: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
+// ARM64: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
+// ARM64: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
+// ARM64: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM64: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
+// ARM64: %[[NOTMASK:[^ ]*]] = xor i8 %[[MASK]], -1
+// ARM64: %[[BYTE:[^ ]*]] = atomicrmw and ptr %[[BYTEADDR]], i8 %[[NOTMASK]] acquire, align 1
+// ARM64: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM64: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM64: store volatile i8 %[[RES]], ptr @sink, align 1
+// ARM64: atomicrmw and ptr %{{.*}}, i8 {{.*}} release, align 1
+// ARM64: atomicrmw and ptr %{{.*}}, i8 {{.*}} monotonic, align 1
+// ARM64: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
+// ARM64: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
+// ARM64: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
+// ARM64: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
+// ARM64: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
+// ARM64: %[[BYTE:[^ ]*]] = atomicrmw or ptr %[[BYTEADDR]], i8 %[[MASK]] acquire, align 1
+// ARM64: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
+// ARM64: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
+// ARM64: store volatile i8 %[[RES]], ptr @sink, align 1
+// ARM64: atomicrmw or ptr %{{.*}}, i8 {{.*}} release, align 1
+// ARM64: atomicrmw or ptr %{{.*}}, i8 {{.*}} monotonic, align 1
|
Contributor
Author
|
@dpaoliello can you fix the labeling eg. add backend:AArch64 |
dpaoliello
approved these changes
Jun 27, 2025
Collaborator
|
LLVM Buildbot has detected a new failure on builder Full details are available at: https://lab.llvm.org/buildbot/#/builders/59/builds/20048 Here is the relevant piece of the build log for the reference |
Contributor
Author
|
I do not see a connection between failure and PR. This is an LLDB failure. |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
5 participants
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Adds interlockedbittestand{set,reset}64{acq,rel,nf} support for AArch64
Includes test changes and has been clang-format-ed
[email protected]
[email protected]
@dpaoliello