Skip to content

Commit bcb857e

Browse files
AdamGlassAdamGlass
committed
interlockedbittestand{set,reset}64_{acq,rel,nf} support for AArch64
1 parent 6bdfeca commit bcb857e

File tree

5 files changed

+202
-6
lines changed

5 files changed

+202
-6
lines changed

‎clang/include/clang/Basic/Builtins.td‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2588,6 +2588,24 @@ def InterlockedBittestAndReset_rel : MSLangBuiltin {
25882588
let Prototype = "unsigned char(msint32_t volatile*, msint32_t)";
25892589
}
25902590

2591+
def InterlockedBittestAndReset64_acq : MSLangBuiltin {
2592+
let Spellings = ["_interlockedbittestandreset64_acq"];
2593+
let Attributes = [NoThrow];
2594+
let Prototype = "unsigned char(int64_t volatile*, int64_t)";
2595+
}
2596+
2597+
def InterlockedBittestAndReset64_nf : MSLangBuiltin {
2598+
let Spellings = ["_interlockedbittestandreset64_nf"];
2599+
let Attributes = [NoThrow];
2600+
let Prototype = "unsigned char(int64_t volatile*, int64_t)";
2601+
}
2602+
2603+
def InterlockedBittestAndReset64_rel : MSLangBuiltin {
2604+
let Spellings = ["_interlockedbittestandreset64_rel"];
2605+
let Attributes = [NoThrow];
2606+
let Prototype = "unsigned char(int64_t volatile*, int64_t)";
2607+
}
2608+
25912609
def InterlockedBittestAndSet : MSLangBuiltin, MSInt32_64Template {
25922610
let Spellings = ["_interlockedbittestandset"];
25932611
let Attributes = [NoThrow];
@@ -2612,6 +2630,24 @@ def InterlockedBittestAndSet_rel : MSLangBuiltin {
26122630
let Prototype = "unsigned char(msint32_t volatile*, msint32_t)";
26132631
}
26142632

2633+
def InterlockedBittestAndSet64_acq : MSLangBuiltin {
2634+
let Spellings = ["_interlockedbittestandset64_acq"];
2635+
let Attributes = [NoThrow];
2636+
let Prototype = "unsigned char(int64_t volatile*, int64_t)";
2637+
}
2638+
2639+
def InterlockedBittestAndSet64_nf : MSLangBuiltin {
2640+
let Spellings = ["_interlockedbittestandset64_nf"];
2641+
let Attributes = [NoThrow];
2642+
let Prototype = "unsigned char(int64_t volatile*, int64_t)";
2643+
}
2644+
2645+
def InterlockedBittestAndSet64_rel : MSLangBuiltin {
2646+
let Spellings = ["_interlockedbittestandset64_rel"];
2647+
let Attributes = [NoThrow];
2648+
let Prototype = "unsigned char(int64_t volatile*, int64_t)";
2649+
}
2650+
26152651
def IsoVolatileLoad : MSLangBuiltin, Int8_16_32_64Template {
26162652
let Spellings = ["__iso_volatile_load"];
26172653
let Attributes = [NoThrow];

‎clang/lib/CodeGen/CGBuiltin.cpp‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1589,7 +1589,7 @@ BitTest BitTest::decodeBitTestBuiltin(unsigned BuiltinID) {
15891589
case Builtin::BI_interlockedbittestandset:
15901590
return {Set, Sequential, false};
15911591

1592-
// X86-specific 64-bit variants.
1592+
// 64-bit variants.
15931593
case Builtin::BI_bittest64:
15941594
return {TestOnly, Unlocked, true};
15951595
case Builtin::BI_bittestandcomplement64:
@@ -1616,6 +1616,18 @@ BitTest BitTest::decodeBitTestBuiltin(unsigned BuiltinID) {
16161616
return {Reset, Release, false};
16171617
case Builtin::BI_interlockedbittestandreset_nf:
16181618
return {Reset, NoFence, false};
1619+
case Builtin::BI_interlockedbittestandreset64_acq:
1620+
return {Reset, Acquire, false};
1621+
case Builtin::BI_interlockedbittestandreset64_rel:
1622+
return {Reset, Release, false};
1623+
case Builtin::BI_interlockedbittestandreset64_nf:
1624+
return {Reset, NoFence, false};
1625+
case Builtin::BI_interlockedbittestandset64_acq:
1626+
return {Set, Acquire, false};
1627+
case Builtin::BI_interlockedbittestandset64_rel:
1628+
return {Set, Release, false};
1629+
case Builtin::BI_interlockedbittestandset64_nf:
1630+
return {Set, NoFence, false};
16191631
}
16201632
llvm_unreachable("expected only bittest intrinsics");
16211633
}
@@ -5538,7 +5550,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
55385550
case Builtin::BI_bittestandset:
55395551
case Builtin::BI_interlockedbittestandreset:
55405552
case Builtin::BI_interlockedbittestandreset64:
5553+
case Builtin::BI_interlockedbittestandreset64_acq:
5554+
case Builtin::BI_interlockedbittestandreset64_rel:
5555+
case Builtin::BI_interlockedbittestandreset64_nf:
55415556
case Builtin::BI_interlockedbittestandset64:
5557+
case Builtin::BI_interlockedbittestandset64_acq:
5558+
case Builtin::BI_interlockedbittestandset64_rel:
5559+
case Builtin::BI_interlockedbittestandset64_nf:
55425560
case Builtin::BI_interlockedbittestandset:
55435561
case Builtin::BI_interlockedbittestandset_acq:
55445562
case Builtin::BI_interlockedbittestandset_rel:

‎clang/lib/Headers/intrin.h‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -370,6 +370,18 @@ static __inline__ void __DEFAULT_FN_ATTRS __nop(void) {
370370
\*----------------------------------------------------------------------------*/
371371
#if defined(__aarch64__) || defined(__arm64ec__)
372372
unsigned __int64 __getReg(int);
373+
unsigned char _interlockedbittestandreset_acq(long volatile *, long);
374+
unsigned char _interlockedbittestandreset_nf(long volatile *, long);
375+
unsigned char _interlockedbittestandreset_rel(long volatile *, long);
376+
unsigned char _interlockedbittestandreset64_acq(__int64 volatile *, __int64);
377+
unsigned char _interlockedbittestandreset64_nf(__int64 volatile *, __int64);
378+
unsigned char _interlockedbittestandreset64_rel(__int64 volatile *, __int64);
379+
unsigned char _interlockedbittestandset_acq(long volatile *, long);
380+
unsigned char _interlockedbittestandset_nf(long volatile *, long);
381+
unsigned char _interlockedbittestandset_rel(long volatile *, long);
382+
unsigned char _interlockedbittestandset64_acq(__int64 volatile *, __int64);
383+
unsigned char _interlockedbittestandset64_nf(__int64 volatile *, __int64);
384+
unsigned char _interlockedbittestandset64_rel(__int64 volatile *, __int64);
373385
long _InterlockedAdd(long volatile *, long);
374386
long _InterlockedAdd_acq(long volatile *, long);
375387
long _InterlockedAdd_nf(long volatile *, long);

‎clang/lib/Sema/SemaChecking.cpp‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2364,6 +2364,17 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
23642364
return ExprError();
23652365
break;
23662366

2367+
// The 64-bit acquire, release, and no fence variants are AArch64 only.
2368+
case Builtin::BI_interlockedbittestandreset64_acq:
2369+
case Builtin::BI_interlockedbittestandreset64_rel:
2370+
case Builtin::BI_interlockedbittestandreset64_nf:
2371+
case Builtin::BI_interlockedbittestandset64_acq:
2372+
case Builtin::BI_interlockedbittestandset64_rel:
2373+
case Builtin::BI_interlockedbittestandset64_nf:
2374+
if (CheckBuiltinTargetInSupported(*this, TheCall, {llvm::Triple::aarch64}))
2375+
return ExprError();
2376+
break;
2377+
23672378
case Builtin::BI__builtin_set_flt_rounds:
23682379
if (CheckBuiltinTargetInSupported(
23692380
*this, TheCall,

‎clang/test/CodeGen/bittest-intrin.c‎

Lines changed: 124 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
// RUN: %clang_cc1 -fms-extensions -triple x86_64-windows-msvc %s -emit-llvm -o - | FileCheck %s --check-prefix=X64
22
// RUN: %clang_cc1 -fms-extensions -triple thumbv7-windows-msvc %s -emit-llvm -o - | FileCheck %s --check-prefix=ARM
3-
// RUN: %clang_cc1 -fms-extensions -triple aarch64-windows-msvc %s -emit-llvm -o - | FileCheck %s --check-prefix=ARM
3+
// RUN: %clang_cc1 -fms-extensions -triple aarch64-windows-msvc %s -emit-llvm -o - | FileCheck %s --check-prefix=ARM64 -check-prefix=ARM
44

55
volatile unsigned char sink = 0;
66
void test32(long *base, long idx) {
@@ -10,7 +10,6 @@ void test32(long *base, long idx) {
1010
sink = _bittestandset(base, idx);
1111
sink = _interlockedbittestandreset(base, idx);
1212
sink = _interlockedbittestandset(base, idx);
13-
sink = _interlockedbittestandset(base, idx);
1413
}
1514

1615
void test64(__int64 *base, __int64 idx) {
@@ -33,6 +32,17 @@ void test_arm(long *base, long idx) {
3332
}
3433
#endif
3534

35+
#if defined(_M_ARM64)
36+
void test_arm64(__int64 *base, __int64 idx) {
37+
sink = _interlockedbittestandreset64_acq(base, idx);
38+
sink = _interlockedbittestandreset64_rel(base, idx);
39+
sink = _interlockedbittestandreset64_nf(base, idx);
40+
sink = _interlockedbittestandset64_acq(base, idx);
41+
sink = _interlockedbittestandset64_rel(base, idx);
42+
sink = _interlockedbittestandset64_nf(base, idx);
43+
}
44+
#endif
45+
3646
// X64-LABEL: define dso_local void @test32(ptr noundef %base, i32 noundef %idx)
3747
// X64: call i8 asm sideeffect "btl $2, ($1)", "={@ccc},r,r,~{cc},~{memory},~{dirflag},~{fpsr},~{flags}"(ptr %{{.*}}, i32 {{.*}})
3848
// X64: call i8 asm sideeffect "btcl $2, ($1)", "={@ccc},r,r,~{cc},~{memory},~{dirflag},~{fpsr},~{flags}"(ptr %{{.*}}, i32 {{.*}})
@@ -117,13 +127,122 @@ void test_arm(long *base, long idx) {
117127
// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
118128
// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
119129

130+
// ARM-LABEL: define dso_local {{.*}}void @test64(ptr noundef %base, i64 noundef %idx)
131+
// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
132+
// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
133+
// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
134+
// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
135+
// ARM: %[[BYTE:[^ ]*]] = load i8, ptr %[[BYTEADDR]], align 1
136+
// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
137+
// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
138+
// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
120139

121-
// Just look for the atomicrmw instructions.
140+
// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
141+
// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
142+
// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
143+
// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
144+
// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
145+
// ARM: %[[BYTE:[^ ]*]] = load i8, ptr %[[BYTEADDR]], align 1
146+
// ARM: %[[NEWBYTE:[^ ]*]] = xor i8 %[[BYTE]], %[[MASK]]
147+
// ARM: store i8 %[[NEWBYTE]], ptr %[[BYTEADDR]], align 1
148+
// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
149+
// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
150+
// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
151+
152+
// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
153+
// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
154+
// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
155+
// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
156+
// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
157+
// ARM: %[[BYTE:[^ ]*]] = load i8, ptr %[[BYTEADDR]], align 1
158+
// ARM: %[[NOTMASK:[^ ]*]] = xor i8 %[[MASK]], -1
159+
// ARM: %[[NEWBYTE:[^ ]*]] = and i8 %[[BYTE]], %[[NOTMASK]]
160+
// ARM: store i8 %[[NEWBYTE]], ptr %[[BYTEADDR]], align 1
161+
// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
162+
// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
163+
// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
164+
165+
// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
166+
// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
167+
// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
168+
// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
169+
// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
170+
// ARM: %[[BYTE:[^ ]*]] = load i8, ptr %[[BYTEADDR]], align 1
171+
// ARM: %[[NEWBYTE:[^ ]*]] = or i8 %[[BYTE]], %[[MASK]]
172+
// ARM: store i8 %[[NEWBYTE]], ptr %[[BYTEADDR]], align 1
173+
// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
174+
// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
175+
// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
176+
177+
// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
178+
// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
179+
// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
180+
// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
181+
// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
182+
// ARM: %[[NOTMASK:[^ ]*]] = xor i8 %[[MASK]], -1
183+
// ARM: %[[BYTE:[^ ]*]] = atomicrmw and ptr %[[BYTEADDR]], i8 %[[NOTMASK]] seq_cst, align 1
184+
// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
185+
// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
186+
// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
187+
188+
// ARM: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
189+
// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
190+
// ARM: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
191+
// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
192+
// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
193+
// ARM: %[[BYTE:[^ ]*]] = atomicrmw or ptr %[[BYTEADDR]], i8 %[[MASK]] seq_cst, align 1
194+
// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
195+
// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
196+
// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
122197

123198
// ARM-LABEL: define dso_local {{.*}}void @test_arm(ptr noundef %base, i32 noundef %idx)
124-
// ARM: atomicrmw and ptr %{{.*}}, i8 {{.*}} acquire, align 1
199+
// ARM: %[[IDXHI:[^ ]*]] = ashr i32 %{{.*}}, 3
200+
// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i32 %[[IDXHI]]
201+
// ARM: %[[IDX8:[^ ]*]] = trunc i32 %{{.*}} to i8
202+
// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
203+
// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
204+
// ARM: %[[NOTMASK:[^ ]*]] = xor i8 %[[MASK]], -1
205+
// ARM: %[[BYTE:[^ ]*]] = atomicrmw and ptr %[[BYTEADDR]], i8 %[[NOTMASK]] acquire, align 1
206+
// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
207+
// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
208+
// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
209+
// Just look for the atomicrmw instructions.
125210
// ARM: atomicrmw and ptr %{{.*}}, i8 {{.*}} release, align 1
126211
// ARM: atomicrmw and ptr %{{.*}}, i8 {{.*}} monotonic, align 1
127-
// ARM: atomicrmw or ptr %{{.*}}, i8 {{.*}} acquire, align 1
212+
// ARM: %[[IDXHI:[^ ]*]] = ashr i32 %{{.*}}, 3
213+
// ARM: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i32 %[[IDXHI]]
214+
// ARM: %[[IDX8:[^ ]*]] = trunc i32 %{{.*}} to i8
215+
// ARM: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
216+
// ARM: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
217+
// ARM: %[[BYTE:[^ ]*]] = atomicrmw or ptr %[[BYTEADDR]], i8 %[[MASK]] acquire, align 1
218+
// ARM: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
219+
// ARM: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
220+
// ARM: store volatile i8 %[[RES]], ptr @sink, align 1
221+
// Just look for the atomicrmw instructions.
128222
// ARM: atomicrmw or ptr %{{.*}}, i8 {{.*}} release, align 1
129223
// ARM: atomicrmw or ptr %{{.*}}, i8 {{.*}} monotonic, align 1
224+
225+
// ARM64-LABEL: define dso_local void @test_arm64(ptr noundef %base, i64 noundef %idx)
226+
// ARM64: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
227+
// ARM64: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
228+
// ARM64: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
229+
// ARM64: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
230+
// ARM64: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
231+
// ARM64: %[[NOTMASK:[^ ]*]] = xor i8 %[[MASK]], -1
232+
// ARM64: %[[BYTE:[^ ]*]] = atomicrmw and ptr %[[BYTEADDR]], i8 %[[NOTMASK]] acquire, align 1
233+
// ARM64: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
234+
// ARM64: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
235+
// ARM64: store volatile i8 %[[RES]], ptr @sink, align 1
236+
// ARM64: atomicrmw and ptr %{{.*}}, i8 {{.*}} release, align 1
237+
// ARM64: atomicrmw and ptr %{{.*}}, i8 {{.*}} monotonic, align 1
238+
// ARM64: %[[IDXHI:[^ ]*]] = ashr i64 %{{.*}}, 3
239+
// ARM64: %[[BYTEADDR:[^ ]*]] = getelementptr inbounds i8, ptr %{{.*}}, i64 %[[IDXHI]]
240+
// ARM64: %[[IDX8:[^ ]*]] = trunc i64 %{{.*}} to i8
241+
// ARM64: %[[IDXLO:[^ ]*]] = and i8 %[[IDX8]], 7
242+
// ARM64: %[[MASK:[^ ]*]] = shl i8 1, %[[IDXLO]]
243+
// ARM64: %[[BYTE:[^ ]*]] = atomicrmw or ptr %[[BYTEADDR]], i8 %[[MASK]] acquire, align 1
244+
// ARM64: %[[BYTESHR:[^ ]*]] = lshr i8 %[[BYTE]], %[[IDXLO]]
245+
// ARM64: %[[RES:[^ ]*]] = and i8 %[[BYTESHR]], 1
246+
// ARM64: store volatile i8 %[[RES]], ptr @sink, align 1
247+
// ARM64: atomicrmw or ptr %{{.*}}, i8 {{.*}} release, align 1
248+
// ARM64: atomicrmw or ptr %{{.*}}, i8 {{.*}} monotonic, align 1

0 commit comments

Comments
 (0)