[aarch64][arm] Add support for the _Interlocked[Compare]ExchangePointer_{acq|nf|rel} MS intrinsics #117645
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
llvmbot
added
clang
Member
|
@llvm/pr-subscribers-clang @llvm/pr-subscribers-backend-aarch64 Author: Daniel Paoliello (dpaoliello) ChangesAdds support for the following MSVC intrinsics:
These are documented at: <https://learn.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics?view=msvc-170#interlocked-intrinsics> NOTE: This change also unifies how the pointer and non-pointer interlocked compare-exchange intrinsics are being handled. Full diff: https://github.com/llvm/llvm-project/pull/117645.diff 5 Files Affected:
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.def b/clang/include/clang/Basic/BuiltinsAArch64.def
index 473b1d4698f04a..486a6d41dbf6a7 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.def
+++ b/clang/include/clang/Basic/BuiltinsAArch64.def
@@ -182,6 +182,9 @@ TARGET_HEADER_BUILTIN(_InterlockedExchange_rel, "NiNiD*Ni", "nh", INTRIN_H,
TARGET_HEADER_BUILTIN(_InterlockedExchange64_acq, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchange64_nf, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchange64_rel, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_acq, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_nf, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_rel, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_acq, "ccD*cc", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_nf, "ccD*cc", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
@@ -195,6 +198,8 @@ TARGET_HEADER_BUILTIN(_InterlockedCompareExchange_rel, "NiNiD*NiNi", "nh",
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_acq, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_nf, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_rel, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_acq, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_rel, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange128, "UcLLiD*LLiLLiLLi*", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange128_acq,"UcLLiD*LLiLLiLLi*", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
diff --git a/clang/include/clang/Basic/BuiltinsARM.def b/clang/include/clang/Basic/BuiltinsARM.def
index 9ee918cb214758..5a7064a98045e7 100644
--- a/clang/include/clang/Basic/BuiltinsARM.def
+++ b/clang/include/clang/Basic/BuiltinsARM.def
@@ -270,6 +270,9 @@ TARGET_HEADER_BUILTIN(_InterlockedExchange_rel, "NiNiD*Ni", "nh", INTRIN_H,
TARGET_HEADER_BUILTIN(_InterlockedExchange64_acq, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchange64_nf, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchange64_rel, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_acq, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_nf, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_rel, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_acq, "ccD*cc", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_nf, "ccD*cc", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
@@ -283,6 +286,8 @@ TARGET_HEADER_BUILTIN(_InterlockedCompareExchange_rel, "NiNiD*NiNi", "nh",
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_acq, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_nf, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_rel, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_acq, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_rel, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedOr8_acq, "ccD*c", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedOr8_nf, "ccD*c", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 91b70b4fdf3d20..a939d44b451762 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -509,8 +509,15 @@ Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,
Address DestAddr = CheckAtomicAlignment(CGF, E);
- auto *Comparand = CGF.EmitScalarExpr(E->getArg(2));
auto *Exchange = CGF.EmitScalarExpr(E->getArg(1));
+ auto *RTy = Exchange->getType();
+
+ auto *Comparand = CGF.EmitScalarExpr(E->getArg(2));
+
+ if (RTy->isPointerTy()) {
+ Exchange = CGF.Builder.CreatePtrToInt(Exchange, CGF.IntPtrTy);
+ Comparand = CGF.Builder.CreatePtrToInt(Comparand, CGF.IntPtrTy);
+ }
// For Release ordering, the failure ordering should be Monotonic.
auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
@@ -521,10 +528,16 @@ Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,
// blocks the few atomics optimizations that LLVM has. If we want to optimize
// _Interlocked* operations in the future, we will have to remove the volatile
// marker.
- auto *Result = CGF.Builder.CreateAtomicCmpXchg(
+ auto *CmpXchg = CGF.Builder.CreateAtomicCmpXchg(
DestAddr, Comparand, Exchange, SuccessOrdering, FailureOrdering);
- Result->setVolatile(true);
- return CGF.Builder.CreateExtractValue(Result, 0);
+ CmpXchg->setVolatile(true);
+
+ auto *Result = CGF.Builder.CreateExtractValue(CmpXchg, 0);
+ if (RTy->isPointerTy()) {
+ Result = CGF.Builder.CreateIntToPtr(Result, RTy);
+ }
+
+ return Result;
}
// 64-bit Microsoft platforms support 128 bit cmpxchg operations. They are
@@ -1595,6 +1608,7 @@ enum class CodeGenFunction::MSVCIntrin {
_BitScanForward,
_BitScanReverse,
_InterlockedAnd,
+ _InterlockedCompareExchange,
_InterlockedDecrement,
_InterlockedExchange,
_InterlockedExchangeAdd,
@@ -1680,26 +1694,31 @@ translateArmToMsvcIntrin(unsigned BuiltinID) {
case clang::ARM::BI_InterlockedExchange16_acq:
case clang::ARM::BI_InterlockedExchange_acq:
case clang::ARM::BI_InterlockedExchange64_acq:
+ case clang::ARM::BI_InterlockedExchangePointer_acq:
return MSVCIntrin::_InterlockedExchange_acq;
case clang::ARM::BI_InterlockedExchange8_rel:
case clang::ARM::BI_InterlockedExchange16_rel:
case clang::ARM::BI_InterlockedExchange_rel:
case clang::ARM::BI_InterlockedExchange64_rel:
+ case clang::ARM::BI_InterlockedExchangePointer_rel:
return MSVCIntrin::_InterlockedExchange_rel;
case clang::ARM::BI_InterlockedExchange8_nf:
case clang::ARM::BI_InterlockedExchange16_nf:
case clang::ARM::BI_InterlockedExchange_nf:
case clang::ARM::BI_InterlockedExchange64_nf:
+ case clang::ARM::BI_InterlockedExchangePointer_nf:
return MSVCIntrin::_InterlockedExchange_nf;
case clang::ARM::BI_InterlockedCompareExchange8_acq:
case clang::ARM::BI_InterlockedCompareExchange16_acq:
case clang::ARM::BI_InterlockedCompareExchange_acq:
case clang::ARM::BI_InterlockedCompareExchange64_acq:
+ case clang::ARM::BI_InterlockedCompareExchangePointer_acq:
return MSVCIntrin::_InterlockedCompareExchange_acq;
case clang::ARM::BI_InterlockedCompareExchange8_rel:
case clang::ARM::BI_InterlockedCompareExchange16_rel:
case clang::ARM::BI_InterlockedCompareExchange_rel:
case clang::ARM::BI_InterlockedCompareExchange64_rel:
+ case clang::ARM::BI_InterlockedCompareExchangePointer_rel:
return MSVCIntrin::_InterlockedCompareExchange_rel;
case clang::ARM::BI_InterlockedCompareExchange8_nf:
case clang::ARM::BI_InterlockedCompareExchange16_nf:
@@ -1826,26 +1845,31 @@ translateAarch64ToMsvcIntrin(unsigned BuiltinID) {
case clang::AArch64::BI_InterlockedExchange16_acq:
case clang::AArch64::BI_InterlockedExchange_acq:
case clang::AArch64::BI_InterlockedExchange64_acq:
+ case clang::AArch64::BI_InterlockedExchangePointer_acq:
return MSVCIntrin::_InterlockedExchange_acq;
case clang::AArch64::BI_InterlockedExchange8_rel:
case clang::AArch64::BI_InterlockedExchange16_rel:
case clang::AArch64::BI_InterlockedExchange_rel:
case clang::AArch64::BI_InterlockedExchange64_rel:
+ case clang::AArch64::BI_InterlockedExchangePointer_rel:
return MSVCIntrin::_InterlockedExchange_rel;
case clang::AArch64::BI_InterlockedExchange8_nf:
case clang::AArch64::BI_InterlockedExchange16_nf:
case clang::AArch64::BI_InterlockedExchange_nf:
case clang::AArch64::BI_InterlockedExchange64_nf:
+ case clang::AArch64::BI_InterlockedExchangePointer_nf:
return MSVCIntrin::_InterlockedExchange_nf;
case clang::AArch64::BI_InterlockedCompareExchange8_acq:
case clang::AArch64::BI_InterlockedCompareExchange16_acq:
case clang::AArch64::BI_InterlockedCompareExchange_acq:
case clang::AArch64::BI_InterlockedCompareExchange64_acq:
+ case clang::AArch64::BI_InterlockedCompareExchangePointer_acq:
return MSVCIntrin::_InterlockedCompareExchange_acq;
case clang::AArch64::BI_InterlockedCompareExchange8_rel:
case clang::AArch64::BI_InterlockedCompareExchange16_rel:
case clang::AArch64::BI_InterlockedCompareExchange_rel:
case clang::AArch64::BI_InterlockedCompareExchange64_rel:
+ case clang::AArch64::BI_InterlockedCompareExchangePointer_rel:
return MSVCIntrin::_InterlockedCompareExchange_rel;
case clang::AArch64::BI_InterlockedCompareExchange8_nf:
case clang::AArch64::BI_InterlockedCompareExchange16_nf:
@@ -2048,6 +2072,8 @@ Value *CodeGenFunction::EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID,
case MSVCIntrin::_InterlockedExchange_nf:
return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E,
AtomicOrdering::Monotonic);
+ case MSVCIntrin::_InterlockedCompareExchange:
+ return EmitAtomicCmpXchgForMSIntrin(*this, E);
case MSVCIntrin::_InterlockedCompareExchange_acq:
return EmitAtomicCmpXchgForMSIntrin(*this, E, AtomicOrdering::Acquire);
case MSVCIntrin::_InterlockedCompareExchange_rel:
@@ -5695,32 +5721,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
return RValue::get(
EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E));
case Builtin::BI_InterlockedCompareExchangePointer:
- case Builtin::BI_InterlockedCompareExchangePointer_nf: {
- llvm::Type *RTy;
- llvm::IntegerType *IntType = IntegerType::get(
- getLLVMContext(), getContext().getTypeSize(E->getType()));
-
- Address DestAddr = CheckAtomicAlignment(*this, E);
-
- llvm::Value *Exchange = EmitScalarExpr(E->getArg(1));
- RTy = Exchange->getType();
- Exchange = Builder.CreatePtrToInt(Exchange, IntType);
-
- llvm::Value *Comparand =
- Builder.CreatePtrToInt(EmitScalarExpr(E->getArg(2)), IntType);
-
- auto Ordering =
- BuiltinID == Builtin::BI_InterlockedCompareExchangePointer_nf ?
- AtomicOrdering::Monotonic : AtomicOrdering::SequentiallyConsistent;
-
- auto Result = Builder.CreateAtomicCmpXchg(DestAddr, Comparand, Exchange,
- Ordering, Ordering);
- Result->setVolatile(true);
-
- return RValue::get(Builder.CreateIntToPtr(Builder.CreateExtractValue(Result,
- 0),
- RTy));
- }
+ return RValue::get(
+ EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange, E));
+ case Builtin::BI_InterlockedCompareExchangePointer_nf:
+ return RValue::get(
+ EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange_nf, E));
case Builtin::BI_InterlockedCompareExchange8:
case Builtin::BI_InterlockedCompareExchange16:
case Builtin::BI_InterlockedCompareExchange:
diff --git a/clang/lib/Headers/intrin0.h b/clang/lib/Headers/intrin0.h
index 6b01f3808652aa..2bca9fc877e9cd 100644
--- a/clang/lib/Headers/intrin0.h
+++ b/clang/lib/Headers/intrin0.h
@@ -207,6 +207,9 @@ long _InterlockedExchange_rel(long volatile *_Target, long _Value);
__int64 _InterlockedExchange64_acq(__int64 volatile *_Target, __int64 _Value);
__int64 _InterlockedExchange64_nf(__int64 volatile *_Target, __int64 _Value);
__int64 _InterlockedExchange64_rel(__int64 volatile *_Target, __int64 _Value);
+void *_InterlockedExchangePointer_acq(void *volatile *_Target, void *_Value);
+void *_InterlockedExchangePointer_nf(void *volatile *_Target, void *_Value);
+void *_InterlockedExchangePointer_rel(void *volatile *_Target, void *_Value);
/*----------------------------------------------------------------------------*\
|* Interlocked Compare Exchange
@@ -237,6 +240,12 @@ __int64 _InterlockedCompareExchange64_nf(__int64 volatile *_Destination,
__int64 _InterlockedCompareExchange64_rel(__int64 volatile *_Destination,
__int64 _Exchange,
__int64 _Comparand);
+void *_InterlockedCompareExchangePointer_acq(void *volatile *_Destination,
+ void *_Exchange, void *_Comparand);
+void *_InterlockedCompareExchangePointer_nf(void *volatile *_Destination,
+ void *_Exchange, void *_Comparand);
+void *_InterlockedCompareExchangePointer_rel(void *volatile *_Destination,
+ void *_Exchange, void *_Comparand);
#endif
#ifdef __cplusplus
diff --git a/clang/test/CodeGen/ms-intrinsics.c b/clang/test/CodeGen/ms-intrinsics.c
index 459a708d9b2e0a..bb1b95fa7521a9 100644
--- a/clang/test/CodeGen/ms-intrinsics.c
+++ b/clang/test/CodeGen/ms-intrinsics.c
@@ -221,6 +221,41 @@ void *test_InterlockedExchangePointer(void * volatile *Target, void *Value) {
// CHECK: ret ptr %[[RESULT]]
// CHECK: }
+#if defined(__arm__) || defined(__aarch64__)
+void *test_InterlockedExchangePointer_acq(void * volatile *Target, void *Value) {
+ return _InterlockedExchangePointer_acq(Target, Value);
+}
+
+// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedExchangePointer_acq(ptr {{[a-z_ ]*}}%Target, ptr {{[a-z_ ]*}}%Value){{.*}}{
+// CHECK-ARM-ARM64: %[[VALUE:[0-9]+]] = ptrtoint ptr %Value to [[iPTR:i[0-9]+]]
+// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = atomicrmw xchg ptr %Target, [[iPTR]] %[[VALUE]] acquire, align {{4|8}}
+// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXCHANGE]] to ptr
+// CHECK-ARM-ARM64: ret ptr %[[RESULT]]
+// CHECK-ARM-ARM64: }
+
+void *test_InterlockedExchangePointer_nf(void * volatile *Target, void *Value) {
+ return _InterlockedExchangePointer_nf(Target, Value);
+}
+
+// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedExchangePointer_nf(ptr {{[a-z_ ]*}}%Target, ptr {{[a-z_ ]*}}%Value){{.*}}{
+// CHECK-ARM-ARM64: %[[VALUE:[0-9]+]] = ptrtoint ptr %Value to [[iPTR]]
+// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = atomicrmw xchg ptr %Target, [[iPTR]] %[[VALUE]] monotonic, align {{4|8}}
+// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXCHANGE]] to ptr
+// CHECK-ARM-ARM64: ret ptr %[[RESULT]]
+// CHECK-ARM-ARM64: }
+
+void *test_InterlockedExchangePointer_rel(void * volatile *Target, void *Value) {
+ return _InterlockedExchangePointer_rel(Target, Value);
+}
+
+// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedExchangePointer_rel(ptr {{[a-z_ ]*}}%Target, ptr {{[a-z_ ]*}}%Value){{.*}}{
+// CHECK-ARM-ARM64: %[[VALUE:[0-9]+]] = ptrtoint ptr %Value to [[iPTR]]
+// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = atomicrmw xchg ptr %Target, [[iPTR]] %[[VALUE]] release, align {{4|8}}
+// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXCHANGE]] to ptr
+// CHECK-ARM-ARM64: ret ptr %[[RESULT]]
+// CHECK-ARM-ARM64: }
+#endif
+
void *test_InterlockedCompareExchangePointer(void * volatile *Destination,
void *Exchange, void *Comparand) {
return _InterlockedCompareExchangePointer(Destination, Exchange, Comparand);
@@ -249,6 +284,37 @@ void *test_InterlockedCompareExchangePointer_nf(void * volatile *Destination,
// CHECK: ret ptr %[[RESULT:[0-9]+]]
// CHECK: }
+#if defined(__arm__) || defined(__aarch64__)
+void *test_InterlockedCompareExchangePointer_acq(void * volatile *Destination,
+ void *Exchange, void *Comparand) {
+ return _InterlockedCompareExchangePointer_acq(Destination, Exchange, Comparand);
+}
+
+// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedCompareExchangePointer_acq(ptr {{[a-z_ ]*}}%Destination, ptr {{[a-z_ ]*}}%Exchange, ptr {{[a-z_ ]*}}%Comparand){{.*}}{
+// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = ptrtoint ptr %Exchange to [[iPTR]]
+// CHECK-ARM-ARM64: %[[COMPARAND:[0-9]+]] = ptrtoint ptr %Comparand to [[iPTR]]
+// CHECK-ARM-ARM64: %[[XCHG:[0-9]+]] = cmpxchg volatile ptr %[[DEST:.+]], [[iPTR]] %[[COMPARAND:[0-9]+]], [[iPTR]] %[[EXCHANGE:[0-9]+]] acquire acquire, align {{4|8}}
+// CHECK-ARM-ARM64: %[[EXTRACT:[0-9]+]] = extractvalue { [[iPTR]], i1 } %[[XCHG]], 0
+// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXTRACT]] to ptr
+// CHECK-ARM-ARM64: ret ptr %[[RESULT:[0-9]+]]
+// CHECK-ARM-ARM64: }
+
+
+void *test_InterlockedCompareExchangePointer_rel(void * volatile *Destination,
+ void *Exchange, void *Comparand) {
+ return _InterlockedCompareExchangePointer_rel(Destination, Exchange, Comparand);
+}
+
+// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedCompareExchangePointer_rel(ptr {{[a-z_ ]*}}%Destination, ptr {{[a-z_ ]*}}%Exchange, ptr {{[a-z_ ]*}}%Comparand){{.*}}{
+// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = ptrtoint ptr %Exchange to [[iPTR]]
+// CHECK-ARM-ARM64: %[[COMPARAND:[0-9]+]] = ptrtoint ptr %Comparand to [[iPTR]]
+// CHECK-ARM-ARM64: %[[XCHG:[0-9]+]] = cmpxchg volatile ptr %[[DEST:.+]], [[iPTR]] %[[COMPARAND:[0-9]+]], [[iPTR]] %[[EXCHANGE:[0-9]+]] release monotonic, align {{4|8}}
+// CHECK-ARM-ARM64: %[[EXTRACT:[0-9]+]] = extractvalue { [[iPTR]], i1 } %[[XCHG]], 0
+// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXTRACT]] to ptr
+// CHECK-ARM-ARM64: ret ptr %[[RESULT:[0-9]+]]
+// CHECK-ARM-ARM64: }
+#endif
+
char test_InterlockedExchange8(char volatile *value, char mask) {
return _InterlockedExchange8(value, mask);
}
|
Member
|
@llvm/pr-subscribers-backend-x86 Author: Daniel Paoliello (dpaoliello) ChangesAdds support for the following MSVC intrinsics:
These are documented at: <https://learn.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics?view=msvc-170#interlocked-intrinsics> NOTE: This change also unifies how the pointer and non-pointer interlocked compare-exchange intrinsics are being handled. Full diff: https://github.com/llvm/llvm-project/pull/117645.diff 5 Files Affected:
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.def b/clang/include/clang/Basic/BuiltinsAArch64.def
index 473b1d4698f04a..486a6d41dbf6a7 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.def
+++ b/clang/include/clang/Basic/BuiltinsAArch64.def
@@ -182,6 +182,9 @@ TARGET_HEADER_BUILTIN(_InterlockedExchange_rel, "NiNiD*Ni", "nh", INTRIN_H,
TARGET_HEADER_BUILTIN(_InterlockedExchange64_acq, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchange64_nf, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchange64_rel, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_acq, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_nf, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_rel, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_acq, "ccD*cc", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_nf, "ccD*cc", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
@@ -195,6 +198,8 @@ TARGET_HEADER_BUILTIN(_InterlockedCompareExchange_rel, "NiNiD*NiNi", "nh",
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_acq, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_nf, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_rel, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_acq, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_rel, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange128, "UcLLiD*LLiLLiLLi*", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange128_acq,"UcLLiD*LLiLLiLLi*", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
diff --git a/clang/include/clang/Basic/BuiltinsARM.def b/clang/include/clang/Basic/BuiltinsARM.def
index 9ee918cb214758..5a7064a98045e7 100644
--- a/clang/include/clang/Basic/BuiltinsARM.def
+++ b/clang/include/clang/Basic/BuiltinsARM.def
@@ -270,6 +270,9 @@ TARGET_HEADER_BUILTIN(_InterlockedExchange_rel, "NiNiD*Ni", "nh", INTRIN_H,
TARGET_HEADER_BUILTIN(_InterlockedExchange64_acq, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchange64_nf, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchange64_rel, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_acq, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_nf, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_rel, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_acq, "ccD*cc", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_nf, "ccD*cc", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
@@ -283,6 +286,8 @@ TARGET_HEADER_BUILTIN(_InterlockedCompareExchange_rel, "NiNiD*NiNi", "nh",
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_acq, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_nf, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_rel, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_acq, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_rel, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedOr8_acq, "ccD*c", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedOr8_nf, "ccD*c", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 91b70b4fdf3d20..a939d44b451762 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -509,8 +509,15 @@ Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,
Address DestAddr = CheckAtomicAlignment(CGF, E);
- auto *Comparand = CGF.EmitScalarExpr(E->getArg(2));
auto *Exchange = CGF.EmitScalarExpr(E->getArg(1));
+ auto *RTy = Exchange->getType();
+
+ auto *Comparand = CGF.EmitScalarExpr(E->getArg(2));
+
+ if (RTy->isPointerTy()) {
+ Exchange = CGF.Builder.CreatePtrToInt(Exchange, CGF.IntPtrTy);
+ Comparand = CGF.Builder.CreatePtrToInt(Comparand, CGF.IntPtrTy);
+ }
// For Release ordering, the failure ordering should be Monotonic.
auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
@@ -521,10 +528,16 @@ Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,
// blocks the few atomics optimizations that LLVM has. If we want to optimize
// _Interlocked* operations in the future, we will have to remove the volatile
// marker.
- auto *Result = CGF.Builder.CreateAtomicCmpXchg(
+ auto *CmpXchg = CGF.Builder.CreateAtomicCmpXchg(
DestAddr, Comparand, Exchange, SuccessOrdering, FailureOrdering);
- Result->setVolatile(true);
- return CGF.Builder.CreateExtractValue(Result, 0);
+ CmpXchg->setVolatile(true);
+
+ auto *Result = CGF.Builder.CreateExtractValue(CmpXchg, 0);
+ if (RTy->isPointerTy()) {
+ Result = CGF.Builder.CreateIntToPtr(Result, RTy);
+ }
+
+ return Result;
}
// 64-bit Microsoft platforms support 128 bit cmpxchg operations. They are
@@ -1595,6 +1608,7 @@ enum class CodeGenFunction::MSVCIntrin {
_BitScanForward,
_BitScanReverse,
_InterlockedAnd,
+ _InterlockedCompareExchange,
_InterlockedDecrement,
_InterlockedExchange,
_InterlockedExchangeAdd,
@@ -1680,26 +1694,31 @@ translateArmToMsvcIntrin(unsigned BuiltinID) {
case clang::ARM::BI_InterlockedExchange16_acq:
case clang::ARM::BI_InterlockedExchange_acq:
case clang::ARM::BI_InterlockedExchange64_acq:
+ case clang::ARM::BI_InterlockedExchangePointer_acq:
return MSVCIntrin::_InterlockedExchange_acq;
case clang::ARM::BI_InterlockedExchange8_rel:
case clang::ARM::BI_InterlockedExchange16_rel:
case clang::ARM::BI_InterlockedExchange_rel:
case clang::ARM::BI_InterlockedExchange64_rel:
+ case clang::ARM::BI_InterlockedExchangePointer_rel:
return MSVCIntrin::_InterlockedExchange_rel;
case clang::ARM::BI_InterlockedExchange8_nf:
case clang::ARM::BI_InterlockedExchange16_nf:
case clang::ARM::BI_InterlockedExchange_nf:
case clang::ARM::BI_InterlockedExchange64_nf:
+ case clang::ARM::BI_InterlockedExchangePointer_nf:
return MSVCIntrin::_InterlockedExchange_nf;
case clang::ARM::BI_InterlockedCompareExchange8_acq:
case clang::ARM::BI_InterlockedCompareExchange16_acq:
case clang::ARM::BI_InterlockedCompareExchange_acq:
case clang::ARM::BI_InterlockedCompareExchange64_acq:
+ case clang::ARM::BI_InterlockedCompareExchangePointer_acq:
return MSVCIntrin::_InterlockedCompareExchange_acq;
case clang::ARM::BI_InterlockedCompareExchange8_rel:
case clang::ARM::BI_InterlockedCompareExchange16_rel:
case clang::ARM::BI_InterlockedCompareExchange_rel:
case clang::ARM::BI_InterlockedCompareExchange64_rel:
+ case clang::ARM::BI_InterlockedCompareExchangePointer_rel:
return MSVCIntrin::_InterlockedCompareExchange_rel;
case clang::ARM::BI_InterlockedCompareExchange8_nf:
case clang::ARM::BI_InterlockedCompareExchange16_nf:
@@ -1826,26 +1845,31 @@ translateAarch64ToMsvcIntrin(unsigned BuiltinID) {
case clang::AArch64::BI_InterlockedExchange16_acq:
case clang::AArch64::BI_InterlockedExchange_acq:
case clang::AArch64::BI_InterlockedExchange64_acq:
+ case clang::AArch64::BI_InterlockedExchangePointer_acq:
return MSVCIntrin::_InterlockedExchange_acq;
case clang::AArch64::BI_InterlockedExchange8_rel:
case clang::AArch64::BI_InterlockedExchange16_rel:
case clang::AArch64::BI_InterlockedExchange_rel:
case clang::AArch64::BI_InterlockedExchange64_rel:
+ case clang::AArch64::BI_InterlockedExchangePointer_rel:
return MSVCIntrin::_InterlockedExchange_rel;
case clang::AArch64::BI_InterlockedExchange8_nf:
case clang::AArch64::BI_InterlockedExchange16_nf:
case clang::AArch64::BI_InterlockedExchange_nf:
case clang::AArch64::BI_InterlockedExchange64_nf:
+ case clang::AArch64::BI_InterlockedExchangePointer_nf:
return MSVCIntrin::_InterlockedExchange_nf;
case clang::AArch64::BI_InterlockedCompareExchange8_acq:
case clang::AArch64::BI_InterlockedCompareExchange16_acq:
case clang::AArch64::BI_InterlockedCompareExchange_acq:
case clang::AArch64::BI_InterlockedCompareExchange64_acq:
+ case clang::AArch64::BI_InterlockedCompareExchangePointer_acq:
return MSVCIntrin::_InterlockedCompareExchange_acq;
case clang::AArch64::BI_InterlockedCompareExchange8_rel:
case clang::AArch64::BI_InterlockedCompareExchange16_rel:
case clang::AArch64::BI_InterlockedCompareExchange_rel:
case clang::AArch64::BI_InterlockedCompareExchange64_rel:
+ case clang::AArch64::BI_InterlockedCompareExchangePointer_rel:
return MSVCIntrin::_InterlockedCompareExchange_rel;
case clang::AArch64::BI_InterlockedCompareExchange8_nf:
case clang::AArch64::BI_InterlockedCompareExchange16_nf:
@@ -2048,6 +2072,8 @@ Value *CodeGenFunction::EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID,
case MSVCIntrin::_InterlockedExchange_nf:
return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E,
AtomicOrdering::Monotonic);
+ case MSVCIntrin::_InterlockedCompareExchange:
+ return EmitAtomicCmpXchgForMSIntrin(*this, E);
case MSVCIntrin::_InterlockedCompareExchange_acq:
return EmitAtomicCmpXchgForMSIntrin(*this, E, AtomicOrdering::Acquire);
case MSVCIntrin::_InterlockedCompareExchange_rel:
@@ -5695,32 +5721,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
return RValue::get(
EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E));
case Builtin::BI_InterlockedCompareExchangePointer:
- case Builtin::BI_InterlockedCompareExchangePointer_nf: {
- llvm::Type *RTy;
- llvm::IntegerType *IntType = IntegerType::get(
- getLLVMContext(), getContext().getTypeSize(E->getType()));
-
- Address DestAddr = CheckAtomicAlignment(*this, E);
-
- llvm::Value *Exchange = EmitScalarExpr(E->getArg(1));
- RTy = Exchange->getType();
- Exchange = Builder.CreatePtrToInt(Exchange, IntType);
-
- llvm::Value *Comparand =
- Builder.CreatePtrToInt(EmitScalarExpr(E->getArg(2)), IntType);
-
- auto Ordering =
- BuiltinID == Builtin::BI_InterlockedCompareExchangePointer_nf ?
- AtomicOrdering::Monotonic : AtomicOrdering::SequentiallyConsistent;
-
- auto Result = Builder.CreateAtomicCmpXchg(DestAddr, Comparand, Exchange,
- Ordering, Ordering);
- Result->setVolatile(true);
-
- return RValue::get(Builder.CreateIntToPtr(Builder.CreateExtractValue(Result,
- 0),
- RTy));
- }
+ return RValue::get(
+ EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange, E));
+ case Builtin::BI_InterlockedCompareExchangePointer_nf:
+ return RValue::get(
+ EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange_nf, E));
case Builtin::BI_InterlockedCompareExchange8:
case Builtin::BI_InterlockedCompareExchange16:
case Builtin::BI_InterlockedCompareExchange:
diff --git a/clang/lib/Headers/intrin0.h b/clang/lib/Headers/intrin0.h
index 6b01f3808652aa..2bca9fc877e9cd 100644
--- a/clang/lib/Headers/intrin0.h
+++ b/clang/lib/Headers/intrin0.h
@@ -207,6 +207,9 @@ long _InterlockedExchange_rel(long volatile *_Target, long _Value);
__int64 _InterlockedExchange64_acq(__int64 volatile *_Target, __int64 _Value);
__int64 _InterlockedExchange64_nf(__int64 volatile *_Target, __int64 _Value);
__int64 _InterlockedExchange64_rel(__int64 volatile *_Target, __int64 _Value);
+void *_InterlockedExchangePointer_acq(void *volatile *_Target, void *_Value);
+void *_InterlockedExchangePointer_nf(void *volatile *_Target, void *_Value);
+void *_InterlockedExchangePointer_rel(void *volatile *_Target, void *_Value);
/*----------------------------------------------------------------------------*\
|* Interlocked Compare Exchange
@@ -237,6 +240,12 @@ __int64 _InterlockedCompareExchange64_nf(__int64 volatile *_Destination,
__int64 _InterlockedCompareExchange64_rel(__int64 volatile *_Destination,
__int64 _Exchange,
__int64 _Comparand);
+void *_InterlockedCompareExchangePointer_acq(void *volatile *_Destination,
+ void *_Exchange, void *_Comparand);
+void *_InterlockedCompareExchangePointer_nf(void *volatile *_Destination,
+ void *_Exchange, void *_Comparand);
+void *_InterlockedCompareExchangePointer_rel(void *volatile *_Destination,
+ void *_Exchange, void *_Comparand);
#endif
#ifdef __cplusplus
diff --git a/clang/test/CodeGen/ms-intrinsics.c b/clang/test/CodeGen/ms-intrinsics.c
index 459a708d9b2e0a..bb1b95fa7521a9 100644
--- a/clang/test/CodeGen/ms-intrinsics.c
+++ b/clang/test/CodeGen/ms-intrinsics.c
@@ -221,6 +221,41 @@ void *test_InterlockedExchangePointer(void * volatile *Target, void *Value) {
// CHECK: ret ptr %[[RESULT]]
// CHECK: }
+#if defined(__arm__) || defined(__aarch64__)
+void *test_InterlockedExchangePointer_acq(void * volatile *Target, void *Value) {
+ return _InterlockedExchangePointer_acq(Target, Value);
+}
+
+// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedExchangePointer_acq(ptr {{[a-z_ ]*}}%Target, ptr {{[a-z_ ]*}}%Value){{.*}}{
+// CHECK-ARM-ARM64: %[[VALUE:[0-9]+]] = ptrtoint ptr %Value to [[iPTR:i[0-9]+]]
+// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = atomicrmw xchg ptr %Target, [[iPTR]] %[[VALUE]] acquire, align {{4|8}}
+// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXCHANGE]] to ptr
+// CHECK-ARM-ARM64: ret ptr %[[RESULT]]
+// CHECK-ARM-ARM64: }
+
+void *test_InterlockedExchangePointer_nf(void * volatile *Target, void *Value) {
+ return _InterlockedExchangePointer_nf(Target, Value);
+}
+
+// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedExchangePointer_nf(ptr {{[a-z_ ]*}}%Target, ptr {{[a-z_ ]*}}%Value){{.*}}{
+// CHECK-ARM-ARM64: %[[VALUE:[0-9]+]] = ptrtoint ptr %Value to [[iPTR]]
+// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = atomicrmw xchg ptr %Target, [[iPTR]] %[[VALUE]] monotonic, align {{4|8}}
+// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXCHANGE]] to ptr
+// CHECK-ARM-ARM64: ret ptr %[[RESULT]]
+// CHECK-ARM-ARM64: }
+
+void *test_InterlockedExchangePointer_rel(void * volatile *Target, void *Value) {
+ return _InterlockedExchangePointer_rel(Target, Value);
+}
+
+// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedExchangePointer_rel(ptr {{[a-z_ ]*}}%Target, ptr {{[a-z_ ]*}}%Value){{.*}}{
+// CHECK-ARM-ARM64: %[[VALUE:[0-9]+]] = ptrtoint ptr %Value to [[iPTR]]
+// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = atomicrmw xchg ptr %Target, [[iPTR]] %[[VALUE]] release, align {{4|8}}
+// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXCHANGE]] to ptr
+// CHECK-ARM-ARM64: ret ptr %[[RESULT]]
+// CHECK-ARM-ARM64: }
+#endif
+
void *test_InterlockedCompareExchangePointer(void * volatile *Destination,
void *Exchange, void *Comparand) {
return _InterlockedCompareExchangePointer(Destination, Exchange, Comparand);
@@ -249,6 +284,37 @@ void *test_InterlockedCompareExchangePointer_nf(void * volatile *Destination,
// CHECK: ret ptr %[[RESULT:[0-9]+]]
// CHECK: }
+#if defined(__arm__) || defined(__aarch64__)
+void *test_InterlockedCompareExchangePointer_acq(void * volatile *Destination,
+ void *Exchange, void *Comparand) {
+ return _InterlockedCompareExchangePointer_acq(Destination, Exchange, Comparand);
+}
+
+// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedCompareExchangePointer_acq(ptr {{[a-z_ ]*}}%Destination, ptr {{[a-z_ ]*}}%Exchange, ptr {{[a-z_ ]*}}%Comparand){{.*}}{
+// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = ptrtoint ptr %Exchange to [[iPTR]]
+// CHECK-ARM-ARM64: %[[COMPARAND:[0-9]+]] = ptrtoint ptr %Comparand to [[iPTR]]
+// CHECK-ARM-ARM64: %[[XCHG:[0-9]+]] = cmpxchg volatile ptr %[[DEST:.+]], [[iPTR]] %[[COMPARAND:[0-9]+]], [[iPTR]] %[[EXCHANGE:[0-9]+]] acquire acquire, align {{4|8}}
+// CHECK-ARM-ARM64: %[[EXTRACT:[0-9]+]] = extractvalue { [[iPTR]], i1 } %[[XCHG]], 0
+// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXTRACT]] to ptr
+// CHECK-ARM-ARM64: ret ptr %[[RESULT:[0-9]+]]
+// CHECK-ARM-ARM64: }
+
+
+void *test_InterlockedCompareExchangePointer_rel(void * volatile *Destination,
+ void *Exchange, void *Comparand) {
+ return _InterlockedCompareExchangePointer_rel(Destination, Exchange, Comparand);
+}
+
+// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedCompareExchangePointer_rel(ptr {{[a-z_ ]*}}%Destination, ptr {{[a-z_ ]*}}%Exchange, ptr {{[a-z_ ]*}}%Comparand){{.*}}{
+// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = ptrtoint ptr %Exchange to [[iPTR]]
+// CHECK-ARM-ARM64: %[[COMPARAND:[0-9]+]] = ptrtoint ptr %Comparand to [[iPTR]]
+// CHECK-ARM-ARM64: %[[XCHG:[0-9]+]] = cmpxchg volatile ptr %[[DEST:.+]], [[iPTR]] %[[COMPARAND:[0-9]+]], [[iPTR]] %[[EXCHANGE:[0-9]+]] release monotonic, align {{4|8}}
+// CHECK-ARM-ARM64: %[[EXTRACT:[0-9]+]] = extractvalue { [[iPTR]], i1 } %[[XCHG]], 0
+// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXTRACT]] to ptr
+// CHECK-ARM-ARM64: ret ptr %[[RESULT:[0-9]+]]
+// CHECK-ARM-ARM64: }
+#endif
+
char test_InterlockedExchange8(char volatile *value, char mask) {
return _InterlockedExchange8(value, mask);
}
|
Member
|
@llvm/pr-subscribers-backend-arm Author: Daniel Paoliello (dpaoliello) ChangesAdds support for the following MSVC intrinsics:
These are documented at: <https://learn.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics?view=msvc-170#interlocked-intrinsics> NOTE: This change also unifies how the pointer and non-pointer interlocked compare-exchange intrinsics are being handled. Full diff: https://github.com/llvm/llvm-project/pull/117645.diff 5 Files Affected:
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.def b/clang/include/clang/Basic/BuiltinsAArch64.def
index 473b1d4698f04a..486a6d41dbf6a7 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.def
+++ b/clang/include/clang/Basic/BuiltinsAArch64.def
@@ -182,6 +182,9 @@ TARGET_HEADER_BUILTIN(_InterlockedExchange_rel, "NiNiD*Ni", "nh", INTRIN_H,
TARGET_HEADER_BUILTIN(_InterlockedExchange64_acq, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchange64_nf, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchange64_rel, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_acq, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_nf, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_rel, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_acq, "ccD*cc", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_nf, "ccD*cc", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
@@ -195,6 +198,8 @@ TARGET_HEADER_BUILTIN(_InterlockedCompareExchange_rel, "NiNiD*NiNi", "nh",
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_acq, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_nf, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_rel, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_acq, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_rel, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange128, "UcLLiD*LLiLLiLLi*", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange128_acq,"UcLLiD*LLiLLiLLi*", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
diff --git a/clang/include/clang/Basic/BuiltinsARM.def b/clang/include/clang/Basic/BuiltinsARM.def
index 9ee918cb214758..5a7064a98045e7 100644
--- a/clang/include/clang/Basic/BuiltinsARM.def
+++ b/clang/include/clang/Basic/BuiltinsARM.def
@@ -270,6 +270,9 @@ TARGET_HEADER_BUILTIN(_InterlockedExchange_rel, "NiNiD*Ni", "nh", INTRIN_H,
TARGET_HEADER_BUILTIN(_InterlockedExchange64_acq, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchange64_nf, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchange64_rel, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_acq, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_nf, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_rel, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_acq, "ccD*cc", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_nf, "ccD*cc", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
@@ -283,6 +286,8 @@ TARGET_HEADER_BUILTIN(_InterlockedCompareExchange_rel, "NiNiD*NiNi", "nh",
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_acq, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_nf, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_rel, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_acq, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_rel, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedOr8_acq, "ccD*c", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedOr8_nf, "ccD*c", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 91b70b4fdf3d20..a939d44b451762 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -509,8 +509,15 @@ Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,
Address DestAddr = CheckAtomicAlignment(CGF, E);
- auto *Comparand = CGF.EmitScalarExpr(E->getArg(2));
auto *Exchange = CGF.EmitScalarExpr(E->getArg(1));
+ auto *RTy = Exchange->getType();
+
+ auto *Comparand = CGF.EmitScalarExpr(E->getArg(2));
+
+ if (RTy->isPointerTy()) {
+ Exchange = CGF.Builder.CreatePtrToInt(Exchange, CGF.IntPtrTy);
+ Comparand = CGF.Builder.CreatePtrToInt(Comparand, CGF.IntPtrTy);
+ }
// For Release ordering, the failure ordering should be Monotonic.
auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
@@ -521,10 +528,16 @@ Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,
// blocks the few atomics optimizations that LLVM has. If we want to optimize
// _Interlocked* operations in the future, we will have to remove the volatile
// marker.
- auto *Result = CGF.Builder.CreateAtomicCmpXchg(
+ auto *CmpXchg = CGF.Builder.CreateAtomicCmpXchg(
DestAddr, Comparand, Exchange, SuccessOrdering, FailureOrdering);
- Result->setVolatile(true);
- return CGF.Builder.CreateExtractValue(Result, 0);
+ CmpXchg->setVolatile(true);
+
+ auto *Result = CGF.Builder.CreateExtractValue(CmpXchg, 0);
+ if (RTy->isPointerTy()) {
+ Result = CGF.Builder.CreateIntToPtr(Result, RTy);
+ }
+
+ return Result;
}
// 64-bit Microsoft platforms support 128 bit cmpxchg operations. They are
@@ -1595,6 +1608,7 @@ enum class CodeGenFunction::MSVCIntrin {
_BitScanForward,
_BitScanReverse,
_InterlockedAnd,
+ _InterlockedCompareExchange,
_InterlockedDecrement,
_InterlockedExchange,
_InterlockedExchangeAdd,
@@ -1680,26 +1694,31 @@ translateArmToMsvcIntrin(unsigned BuiltinID) {
case clang::ARM::BI_InterlockedExchange16_acq:
case clang::ARM::BI_InterlockedExchange_acq:
case clang::ARM::BI_InterlockedExchange64_acq:
+ case clang::ARM::BI_InterlockedExchangePointer_acq:
return MSVCIntrin::_InterlockedExchange_acq;
case clang::ARM::BI_InterlockedExchange8_rel:
case clang::ARM::BI_InterlockedExchange16_rel:
case clang::ARM::BI_InterlockedExchange_rel:
case clang::ARM::BI_InterlockedExchange64_rel:
+ case clang::ARM::BI_InterlockedExchangePointer_rel:
return MSVCIntrin::_InterlockedExchange_rel;
case clang::ARM::BI_InterlockedExchange8_nf:
case clang::ARM::BI_InterlockedExchange16_nf:
case clang::ARM::BI_InterlockedExchange_nf:
case clang::ARM::BI_InterlockedExchange64_nf:
+ case clang::ARM::BI_InterlockedExchangePointer_nf:
return MSVCIntrin::_InterlockedExchange_nf;
case clang::ARM::BI_InterlockedCompareExchange8_acq:
case clang::ARM::BI_InterlockedCompareExchange16_acq:
case clang::ARM::BI_InterlockedCompareExchange_acq:
case clang::ARM::BI_InterlockedCompareExchange64_acq:
+ case clang::ARM::BI_InterlockedCompareExchangePointer_acq:
return MSVCIntrin::_InterlockedCompareExchange_acq;
case clang::ARM::BI_InterlockedCompareExchange8_rel:
case clang::ARM::BI_InterlockedCompareExchange16_rel:
case clang::ARM::BI_InterlockedCompareExchange_rel:
case clang::ARM::BI_InterlockedCompareExchange64_rel:
+ case clang::ARM::BI_InterlockedCompareExchangePointer_rel:
return MSVCIntrin::_InterlockedCompareExchange_rel;
case clang::ARM::BI_InterlockedCompareExchange8_nf:
case clang::ARM::BI_InterlockedCompareExchange16_nf:
@@ -1826,26 +1845,31 @@ translateAarch64ToMsvcIntrin(unsigned BuiltinID) {
case clang::AArch64::BI_InterlockedExchange16_acq:
case clang::AArch64::BI_InterlockedExchange_acq:
case clang::AArch64::BI_InterlockedExchange64_acq:
+ case clang::AArch64::BI_InterlockedExchangePointer_acq:
return MSVCIntrin::_InterlockedExchange_acq;
case clang::AArch64::BI_InterlockedExchange8_rel:
case clang::AArch64::BI_InterlockedExchange16_rel:
case clang::AArch64::BI_InterlockedExchange_rel:
case clang::AArch64::BI_InterlockedExchange64_rel:
+ case clang::AArch64::BI_InterlockedExchangePointer_rel:
return MSVCIntrin::_InterlockedExchange_rel;
case clang::AArch64::BI_InterlockedExchange8_nf:
case clang::AArch64::BI_InterlockedExchange16_nf:
case clang::AArch64::BI_InterlockedExchange_nf:
case clang::AArch64::BI_InterlockedExchange64_nf:
+ case clang::AArch64::BI_InterlockedExchangePointer_nf:
return MSVCIntrin::_InterlockedExchange_nf;
case clang::AArch64::BI_InterlockedCompareExchange8_acq:
case clang::AArch64::BI_InterlockedCompareExchange16_acq:
case clang::AArch64::BI_InterlockedCompareExchange_acq:
case clang::AArch64::BI_InterlockedCompareExchange64_acq:
+ case clang::AArch64::BI_InterlockedCompareExchangePointer_acq:
return MSVCIntrin::_InterlockedCompareExchange_acq;
case clang::AArch64::BI_InterlockedCompareExchange8_rel:
case clang::AArch64::BI_InterlockedCompareExchange16_rel:
case clang::AArch64::BI_InterlockedCompareExchange_rel:
case clang::AArch64::BI_InterlockedCompareExchange64_rel:
+ case clang::AArch64::BI_InterlockedCompareExchangePointer_rel:
return MSVCIntrin::_InterlockedCompareExchange_rel;
case clang::AArch64::BI_InterlockedCompareExchange8_nf:
case clang::AArch64::BI_InterlockedCompareExchange16_nf:
@@ -2048,6 +2072,8 @@ Value *CodeGenFunction::EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID,
case MSVCIntrin::_InterlockedExchange_nf:
return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E,
AtomicOrdering::Monotonic);
+ case MSVCIntrin::_InterlockedCompareExchange:
+ return EmitAtomicCmpXchgForMSIntrin(*this, E);
case MSVCIntrin::_InterlockedCompareExchange_acq:
return EmitAtomicCmpXchgForMSIntrin(*this, E, AtomicOrdering::Acquire);
case MSVCIntrin::_InterlockedCompareExchange_rel:
@@ -5695,32 +5721,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
return RValue::get(
EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E));
case Builtin::BI_InterlockedCompareExchangePointer:
- case Builtin::BI_InterlockedCompareExchangePointer_nf: {
- llvm::Type *RTy;
- llvm::IntegerType *IntType = IntegerType::get(
- getLLVMContext(), getContext().getTypeSize(E->getType()));
-
- Address DestAddr = CheckAtomicAlignment(*this, E);
-
- llvm::Value *Exchange = EmitScalarExpr(E->getArg(1));
- RTy = Exchange->getType();
- Exchange = Builder.CreatePtrToInt(Exchange, IntType);
-
- llvm::Value *Comparand =
- Builder.CreatePtrToInt(EmitScalarExpr(E->getArg(2)), IntType);
-
- auto Ordering =
- BuiltinID == Builtin::BI_InterlockedCompareExchangePointer_nf ?
- AtomicOrdering::Monotonic : AtomicOrdering::SequentiallyConsistent;
-
- auto Result = Builder.CreateAtomicCmpXchg(DestAddr, Comparand, Exchange,
- Ordering, Ordering);
- Result->setVolatile(true);
-
- return RValue::get(Builder.CreateIntToPtr(Builder.CreateExtractValue(Result,
- 0),
- RTy));
- }
+ return RValue::get(
+ EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange, E));
+ case Builtin::BI_InterlockedCompareExchangePointer_nf:
+ return RValue::get(
+ EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange_nf, E));
case Builtin::BI_InterlockedCompareExchange8:
case Builtin::BI_InterlockedCompareExchange16:
case Builtin::BI_InterlockedCompareExchange:
diff --git a/clang/lib/Headers/intrin0.h b/clang/lib/Headers/intrin0.h
index 6b01f3808652aa..2bca9fc877e9cd 100644
--- a/clang/lib/Headers/intrin0.h
+++ b/clang/lib/Headers/intrin0.h
@@ -207,6 +207,9 @@ long _InterlockedExchange_rel(long volatile *_Target, long _Value);
__int64 _InterlockedExchange64_acq(__int64 volatile *_Target, __int64 _Value);
__int64 _InterlockedExchange64_nf(__int64 volatile *_Target, __int64 _Value);
__int64 _InterlockedExchange64_rel(__int64 volatile *_Target, __int64 _Value);
+void *_InterlockedExchangePointer_acq(void *volatile *_Target, void *_Value);
+void *_InterlockedExchangePointer_nf(void *volatile *_Target, void *_Value);
+void *_InterlockedExchangePointer_rel(void *volatile *_Target, void *_Value);
/*----------------------------------------------------------------------------*\
|* Interlocked Compare Exchange
@@ -237,6 +240,12 @@ __int64 _InterlockedCompareExchange64_nf(__int64 volatile *_Destination,
__int64 _InterlockedCompareExchange64_rel(__int64 volatile *_Destination,
__int64 _Exchange,
__int64 _Comparand);
+void *_InterlockedCompareExchangePointer_acq(void *volatile *_Destination,
+ void *_Exchange, void *_Comparand);
+void *_InterlockedCompareExchangePointer_nf(void *volatile *_Destination,
+ void *_Exchange, void *_Comparand);
+void *_InterlockedCompareExchangePointer_rel(void *volatile *_Destination,
+ void *_Exchange, void *_Comparand);
#endif
#ifdef __cplusplus
diff --git a/clang/test/CodeGen/ms-intrinsics.c b/clang/test/CodeGen/ms-intrinsics.c
index 459a708d9b2e0a..bb1b95fa7521a9 100644
--- a/clang/test/CodeGen/ms-intrinsics.c
+++ b/clang/test/CodeGen/ms-intrinsics.c
@@ -221,6 +221,41 @@ void *test_InterlockedExchangePointer(void * volatile *Target, void *Value) {
// CHECK: ret ptr %[[RESULT]]
// CHECK: }
+#if defined(__arm__) || defined(__aarch64__)
+void *test_InterlockedExchangePointer_acq(void * volatile *Target, void *Value) {
+ return _InterlockedExchangePointer_acq(Target, Value);
+}
+
+// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedExchangePointer_acq(ptr {{[a-z_ ]*}}%Target, ptr {{[a-z_ ]*}}%Value){{.*}}{
+// CHECK-ARM-ARM64: %[[VALUE:[0-9]+]] = ptrtoint ptr %Value to [[iPTR:i[0-9]+]]
+// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = atomicrmw xchg ptr %Target, [[iPTR]] %[[VALUE]] acquire, align {{4|8}}
+// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXCHANGE]] to ptr
+// CHECK-ARM-ARM64: ret ptr %[[RESULT]]
+// CHECK-ARM-ARM64: }
+
+void *test_InterlockedExchangePointer_nf(void * volatile *Target, void *Value) {
+ return _InterlockedExchangePointer_nf(Target, Value);
+}
+
+// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedExchangePointer_nf(ptr {{[a-z_ ]*}}%Target, ptr {{[a-z_ ]*}}%Value){{.*}}{
+// CHECK-ARM-ARM64: %[[VALUE:[0-9]+]] = ptrtoint ptr %Value to [[iPTR]]
+// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = atomicrmw xchg ptr %Target, [[iPTR]] %[[VALUE]] monotonic, align {{4|8}}
+// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXCHANGE]] to ptr
+// CHECK-ARM-ARM64: ret ptr %[[RESULT]]
+// CHECK-ARM-ARM64: }
+
+void *test_InterlockedExchangePointer_rel(void * volatile *Target, void *Value) {
+ return _InterlockedExchangePointer_rel(Target, Value);
+}
+
+// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedExchangePointer_rel(ptr {{[a-z_ ]*}}%Target, ptr {{[a-z_ ]*}}%Value){{.*}}{
+// CHECK-ARM-ARM64: %[[VALUE:[0-9]+]] = ptrtoint ptr %Value to [[iPTR]]
+// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = atomicrmw xchg ptr %Target, [[iPTR]] %[[VALUE]] release, align {{4|8}}
+// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXCHANGE]] to ptr
+// CHECK-ARM-ARM64: ret ptr %[[RESULT]]
+// CHECK-ARM-ARM64: }
+#endif
+
void *test_InterlockedCompareExchangePointer(void * volatile *Destination,
void *Exchange, void *Comparand) {
return _InterlockedCompareExchangePointer(Destination, Exchange, Comparand);
@@ -249,6 +284,37 @@ void *test_InterlockedCompareExchangePointer_nf(void * volatile *Destination,
// CHECK: ret ptr %[[RESULT:[0-9]+]]
// CHECK: }
+#if defined(__arm__) || defined(__aarch64__)
+void *test_InterlockedCompareExchangePointer_acq(void * volatile *Destination,
+ void *Exchange, void *Comparand) {
+ return _InterlockedCompareExchangePointer_acq(Destination, Exchange, Comparand);
+}
+
+// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedCompareExchangePointer_acq(ptr {{[a-z_ ]*}}%Destination, ptr {{[a-z_ ]*}}%Exchange, ptr {{[a-z_ ]*}}%Comparand){{.*}}{
+// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = ptrtoint ptr %Exchange to [[iPTR]]
+// CHECK-ARM-ARM64: %[[COMPARAND:[0-9]+]] = ptrtoint ptr %Comparand to [[iPTR]]
+// CHECK-ARM-ARM64: %[[XCHG:[0-9]+]] = cmpxchg volatile ptr %[[DEST:.+]], [[iPTR]] %[[COMPARAND:[0-9]+]], [[iPTR]] %[[EXCHANGE:[0-9]+]] acquire acquire, align {{4|8}}
+// CHECK-ARM-ARM64: %[[EXTRACT:[0-9]+]] = extractvalue { [[iPTR]], i1 } %[[XCHG]], 0
+// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXTRACT]] to ptr
+// CHECK-ARM-ARM64: ret ptr %[[RESULT:[0-9]+]]
+// CHECK-ARM-ARM64: }
+
+
+void *test_InterlockedCompareExchangePointer_rel(void * volatile *Destination,
+ void *Exchange, void *Comparand) {
+ return _InterlockedCompareExchangePointer_rel(Destination, Exchange, Comparand);
+}
+
+// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedCompareExchangePointer_rel(ptr {{[a-z_ ]*}}%Destination, ptr {{[a-z_ ]*}}%Exchange, ptr {{[a-z_ ]*}}%Comparand){{.*}}{
+// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = ptrtoint ptr %Exchange to [[iPTR]]
+// CHECK-ARM-ARM64: %[[COMPARAND:[0-9]+]] = ptrtoint ptr %Comparand to [[iPTR]]
+// CHECK-ARM-ARM64: %[[XCHG:[0-9]+]] = cmpxchg volatile ptr %[[DEST:.+]], [[iPTR]] %[[COMPARAND:[0-9]+]], [[iPTR]] %[[EXCHANGE:[0-9]+]] release monotonic, align {{4|8}}
+// CHECK-ARM-ARM64: %[[EXTRACT:[0-9]+]] = extractvalue { [[iPTR]], i1 } %[[XCHG]], 0
+// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXTRACT]] to ptr
+// CHECK-ARM-ARM64: ret ptr %[[RESULT:[0-9]+]]
+// CHECK-ARM-ARM64: }
+#endif
+
char test_InterlockedExchange8(char volatile *value, char mask) {
return _InterlockedExchange8(value, mask);
}
|
Member
|
@llvm/pr-subscribers-clang-codegen Author: Daniel Paoliello (dpaoliello) ChangesAdds support for the following MSVC intrinsics:
These are documented at: <https://learn.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics?view=msvc-170#interlocked-intrinsics> NOTE: This change also unifies how the pointer and non-pointer interlocked compare-exchange intrinsics are being handled. Full diff: https://github.com/llvm/llvm-project/pull/117645.diff 5 Files Affected:
diff --git a/clang/include/clang/Basic/BuiltinsAArch64.def b/clang/include/clang/Basic/BuiltinsAArch64.def
index 473b1d4698f04a..486a6d41dbf6a7 100644
--- a/clang/include/clang/Basic/BuiltinsAArch64.def
+++ b/clang/include/clang/Basic/BuiltinsAArch64.def
@@ -182,6 +182,9 @@ TARGET_HEADER_BUILTIN(_InterlockedExchange_rel, "NiNiD*Ni", "nh", INTRIN_H,
TARGET_HEADER_BUILTIN(_InterlockedExchange64_acq, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchange64_nf, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchange64_rel, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_acq, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_nf, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_rel, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_acq, "ccD*cc", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_nf, "ccD*cc", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
@@ -195,6 +198,8 @@ TARGET_HEADER_BUILTIN(_InterlockedCompareExchange_rel, "NiNiD*NiNi", "nh",
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_acq, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_nf, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_rel, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_acq, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_rel, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange128, "UcLLiD*LLiLLiLLi*", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange128_acq,"UcLLiD*LLiLLiLLi*", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
diff --git a/clang/include/clang/Basic/BuiltinsARM.def b/clang/include/clang/Basic/BuiltinsARM.def
index 9ee918cb214758..5a7064a98045e7 100644
--- a/clang/include/clang/Basic/BuiltinsARM.def
+++ b/clang/include/clang/Basic/BuiltinsARM.def
@@ -270,6 +270,9 @@ TARGET_HEADER_BUILTIN(_InterlockedExchange_rel, "NiNiD*Ni", "nh", INTRIN_H,
TARGET_HEADER_BUILTIN(_InterlockedExchange64_acq, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchange64_nf, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchange64_rel, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_acq, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_nf, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_rel, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_acq, "ccD*cc", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_nf, "ccD*cc", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
@@ -283,6 +286,8 @@ TARGET_HEADER_BUILTIN(_InterlockedCompareExchange_rel, "NiNiD*NiNi", "nh",
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_acq, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_nf, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_rel, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_acq, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
+TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_rel, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedOr8_acq, "ccD*c", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedOr8_nf, "ccD*c", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 91b70b4fdf3d20..a939d44b451762 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -509,8 +509,15 @@ Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,
Address DestAddr = CheckAtomicAlignment(CGF, E);
- auto *Comparand = CGF.EmitScalarExpr(E->getArg(2));
auto *Exchange = CGF.EmitScalarExpr(E->getArg(1));
+ auto *RTy = Exchange->getType();
+
+ auto *Comparand = CGF.EmitScalarExpr(E->getArg(2));
+
+ if (RTy->isPointerTy()) {
+ Exchange = CGF.Builder.CreatePtrToInt(Exchange, CGF.IntPtrTy);
+ Comparand = CGF.Builder.CreatePtrToInt(Comparand, CGF.IntPtrTy);
+ }
// For Release ordering, the failure ordering should be Monotonic.
auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
@@ -521,10 +528,16 @@ Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,
// blocks the few atomics optimizations that LLVM has. If we want to optimize
// _Interlocked* operations in the future, we will have to remove the volatile
// marker.
- auto *Result = CGF.Builder.CreateAtomicCmpXchg(
+ auto *CmpXchg = CGF.Builder.CreateAtomicCmpXchg(
DestAddr, Comparand, Exchange, SuccessOrdering, FailureOrdering);
- Result->setVolatile(true);
- return CGF.Builder.CreateExtractValue(Result, 0);
+ CmpXchg->setVolatile(true);
+
+ auto *Result = CGF.Builder.CreateExtractValue(CmpXchg, 0);
+ if (RTy->isPointerTy()) {
+ Result = CGF.Builder.CreateIntToPtr(Result, RTy);
+ }
+
+ return Result;
}
// 64-bit Microsoft platforms support 128 bit cmpxchg operations. They are
@@ -1595,6 +1608,7 @@ enum class CodeGenFunction::MSVCIntrin {
_BitScanForward,
_BitScanReverse,
_InterlockedAnd,
+ _InterlockedCompareExchange,
_InterlockedDecrement,
_InterlockedExchange,
_InterlockedExchangeAdd,
@@ -1680,26 +1694,31 @@ translateArmToMsvcIntrin(unsigned BuiltinID) {
case clang::ARM::BI_InterlockedExchange16_acq:
case clang::ARM::BI_InterlockedExchange_acq:
case clang::ARM::BI_InterlockedExchange64_acq:
+ case clang::ARM::BI_InterlockedExchangePointer_acq:
return MSVCIntrin::_InterlockedExchange_acq;
case clang::ARM::BI_InterlockedExchange8_rel:
case clang::ARM::BI_InterlockedExchange16_rel:
case clang::ARM::BI_InterlockedExchange_rel:
case clang::ARM::BI_InterlockedExchange64_rel:
+ case clang::ARM::BI_InterlockedExchangePointer_rel:
return MSVCIntrin::_InterlockedExchange_rel;
case clang::ARM::BI_InterlockedExchange8_nf:
case clang::ARM::BI_InterlockedExchange16_nf:
case clang::ARM::BI_InterlockedExchange_nf:
case clang::ARM::BI_InterlockedExchange64_nf:
+ case clang::ARM::BI_InterlockedExchangePointer_nf:
return MSVCIntrin::_InterlockedExchange_nf;
case clang::ARM::BI_InterlockedCompareExchange8_acq:
case clang::ARM::BI_InterlockedCompareExchange16_acq:
case clang::ARM::BI_InterlockedCompareExchange_acq:
case clang::ARM::BI_InterlockedCompareExchange64_acq:
+ case clang::ARM::BI_InterlockedCompareExchangePointer_acq:
return MSVCIntrin::_InterlockedCompareExchange_acq;
case clang::ARM::BI_InterlockedCompareExchange8_rel:
case clang::ARM::BI_InterlockedCompareExchange16_rel:
case clang::ARM::BI_InterlockedCompareExchange_rel:
case clang::ARM::BI_InterlockedCompareExchange64_rel:
+ case clang::ARM::BI_InterlockedCompareExchangePointer_rel:
return MSVCIntrin::_InterlockedCompareExchange_rel;
case clang::ARM::BI_InterlockedCompareExchange8_nf:
case clang::ARM::BI_InterlockedCompareExchange16_nf:
@@ -1826,26 +1845,31 @@ translateAarch64ToMsvcIntrin(unsigned BuiltinID) {
case clang::AArch64::BI_InterlockedExchange16_acq:
case clang::AArch64::BI_InterlockedExchange_acq:
case clang::AArch64::BI_InterlockedExchange64_acq:
+ case clang::AArch64::BI_InterlockedExchangePointer_acq:
return MSVCIntrin::_InterlockedExchange_acq;
case clang::AArch64::BI_InterlockedExchange8_rel:
case clang::AArch64::BI_InterlockedExchange16_rel:
case clang::AArch64::BI_InterlockedExchange_rel:
case clang::AArch64::BI_InterlockedExchange64_rel:
+ case clang::AArch64::BI_InterlockedExchangePointer_rel:
return MSVCIntrin::_InterlockedExchange_rel;
case clang::AArch64::BI_InterlockedExchange8_nf:
case clang::AArch64::BI_InterlockedExchange16_nf:
case clang::AArch64::BI_InterlockedExchange_nf:
case clang::AArch64::BI_InterlockedExchange64_nf:
+ case clang::AArch64::BI_InterlockedExchangePointer_nf:
return MSVCIntrin::_InterlockedExchange_nf;
case clang::AArch64::BI_InterlockedCompareExchange8_acq:
case clang::AArch64::BI_InterlockedCompareExchange16_acq:
case clang::AArch64::BI_InterlockedCompareExchange_acq:
case clang::AArch64::BI_InterlockedCompareExchange64_acq:
+ case clang::AArch64::BI_InterlockedCompareExchangePointer_acq:
return MSVCIntrin::_InterlockedCompareExchange_acq;
case clang::AArch64::BI_InterlockedCompareExchange8_rel:
case clang::AArch64::BI_InterlockedCompareExchange16_rel:
case clang::AArch64::BI_InterlockedCompareExchange_rel:
case clang::AArch64::BI_InterlockedCompareExchange64_rel:
+ case clang::AArch64::BI_InterlockedCompareExchangePointer_rel:
return MSVCIntrin::_InterlockedCompareExchange_rel;
case clang::AArch64::BI_InterlockedCompareExchange8_nf:
case clang::AArch64::BI_InterlockedCompareExchange16_nf:
@@ -2048,6 +2072,8 @@ Value *CodeGenFunction::EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID,
case MSVCIntrin::_InterlockedExchange_nf:
return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E,
AtomicOrdering::Monotonic);
+ case MSVCIntrin::_InterlockedCompareExchange:
+ return EmitAtomicCmpXchgForMSIntrin(*this, E);
case MSVCIntrin::_InterlockedCompareExchange_acq:
return EmitAtomicCmpXchgForMSIntrin(*this, E, AtomicOrdering::Acquire);
case MSVCIntrin::_InterlockedCompareExchange_rel:
@@ -5695,32 +5721,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
return RValue::get(
EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E));
case Builtin::BI_InterlockedCompareExchangePointer:
- case Builtin::BI_InterlockedCompareExchangePointer_nf: {
- llvm::Type *RTy;
- llvm::IntegerType *IntType = IntegerType::get(
- getLLVMContext(), getContext().getTypeSize(E->getType()));
-
- Address DestAddr = CheckAtomicAlignment(*this, E);
-
- llvm::Value *Exchange = EmitScalarExpr(E->getArg(1));
- RTy = Exchange->getType();
- Exchange = Builder.CreatePtrToInt(Exchange, IntType);
-
- llvm::Value *Comparand =
- Builder.CreatePtrToInt(EmitScalarExpr(E->getArg(2)), IntType);
-
- auto Ordering =
- BuiltinID == Builtin::BI_InterlockedCompareExchangePointer_nf ?
- AtomicOrdering::Monotonic : AtomicOrdering::SequentiallyConsistent;
-
- auto Result = Builder.CreateAtomicCmpXchg(DestAddr, Comparand, Exchange,
- Ordering, Ordering);
- Result->setVolatile(true);
-
- return RValue::get(Builder.CreateIntToPtr(Builder.CreateExtractValue(Result,
- 0),
- RTy));
- }
+ return RValue::get(
+ EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange, E));
+ case Builtin::BI_InterlockedCompareExchangePointer_nf:
+ return RValue::get(
+ EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange_nf, E));
case Builtin::BI_InterlockedCompareExchange8:
case Builtin::BI_InterlockedCompareExchange16:
case Builtin::BI_InterlockedCompareExchange:
diff --git a/clang/lib/Headers/intrin0.h b/clang/lib/Headers/intrin0.h
index 6b01f3808652aa..2bca9fc877e9cd 100644
--- a/clang/lib/Headers/intrin0.h
+++ b/clang/lib/Headers/intrin0.h
@@ -207,6 +207,9 @@ long _InterlockedExchange_rel(long volatile *_Target, long _Value);
__int64 _InterlockedExchange64_acq(__int64 volatile *_Target, __int64 _Value);
__int64 _InterlockedExchange64_nf(__int64 volatile *_Target, __int64 _Value);
__int64 _InterlockedExchange64_rel(__int64 volatile *_Target, __int64 _Value);
+void *_InterlockedExchangePointer_acq(void *volatile *_Target, void *_Value);
+void *_InterlockedExchangePointer_nf(void *volatile *_Target, void *_Value);
+void *_InterlockedExchangePointer_rel(void *volatile *_Target, void *_Value);
/*----------------------------------------------------------------------------*\
|* Interlocked Compare Exchange
@@ -237,6 +240,12 @@ __int64 _InterlockedCompareExchange64_nf(__int64 volatile *_Destination,
__int64 _InterlockedCompareExchange64_rel(__int64 volatile *_Destination,
__int64 _Exchange,
__int64 _Comparand);
+void *_InterlockedCompareExchangePointer_acq(void *volatile *_Destination,
+ void *_Exchange, void *_Comparand);
+void *_InterlockedCompareExchangePointer_nf(void *volatile *_Destination,
+ void *_Exchange, void *_Comparand);
+void *_InterlockedCompareExchangePointer_rel(void *volatile *_Destination,
+ void *_Exchange, void *_Comparand);
#endif
#ifdef __cplusplus
diff --git a/clang/test/CodeGen/ms-intrinsics.c b/clang/test/CodeGen/ms-intrinsics.c
index 459a708d9b2e0a..bb1b95fa7521a9 100644
--- a/clang/test/CodeGen/ms-intrinsics.c
+++ b/clang/test/CodeGen/ms-intrinsics.c
@@ -221,6 +221,41 @@ void *test_InterlockedExchangePointer(void * volatile *Target, void *Value) {
// CHECK: ret ptr %[[RESULT]]
// CHECK: }
+#if defined(__arm__) || defined(__aarch64__)
+void *test_InterlockedExchangePointer_acq(void * volatile *Target, void *Value) {
+ return _InterlockedExchangePointer_acq(Target, Value);
+}
+
+// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedExchangePointer_acq(ptr {{[a-z_ ]*}}%Target, ptr {{[a-z_ ]*}}%Value){{.*}}{
+// CHECK-ARM-ARM64: %[[VALUE:[0-9]+]] = ptrtoint ptr %Value to [[iPTR:i[0-9]+]]
+// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = atomicrmw xchg ptr %Target, [[iPTR]] %[[VALUE]] acquire, align {{4|8}}
+// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXCHANGE]] to ptr
+// CHECK-ARM-ARM64: ret ptr %[[RESULT]]
+// CHECK-ARM-ARM64: }
+
+void *test_InterlockedExchangePointer_nf(void * volatile *Target, void *Value) {
+ return _InterlockedExchangePointer_nf(Target, Value);
+}
+
+// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedExchangePointer_nf(ptr {{[a-z_ ]*}}%Target, ptr {{[a-z_ ]*}}%Value){{.*}}{
+// CHECK-ARM-ARM64: %[[VALUE:[0-9]+]] = ptrtoint ptr %Value to [[iPTR]]
+// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = atomicrmw xchg ptr %Target, [[iPTR]] %[[VALUE]] monotonic, align {{4|8}}
+// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXCHANGE]] to ptr
+// CHECK-ARM-ARM64: ret ptr %[[RESULT]]
+// CHECK-ARM-ARM64: }
+
+void *test_InterlockedExchangePointer_rel(void * volatile *Target, void *Value) {
+ return _InterlockedExchangePointer_rel(Target, Value);
+}
+
+// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedExchangePointer_rel(ptr {{[a-z_ ]*}}%Target, ptr {{[a-z_ ]*}}%Value){{.*}}{
+// CHECK-ARM-ARM64: %[[VALUE:[0-9]+]] = ptrtoint ptr %Value to [[iPTR]]
+// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = atomicrmw xchg ptr %Target, [[iPTR]] %[[VALUE]] release, align {{4|8}}
+// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXCHANGE]] to ptr
+// CHECK-ARM-ARM64: ret ptr %[[RESULT]]
+// CHECK-ARM-ARM64: }
+#endif
+
void *test_InterlockedCompareExchangePointer(void * volatile *Destination,
void *Exchange, void *Comparand) {
return _InterlockedCompareExchangePointer(Destination, Exchange, Comparand);
@@ -249,6 +284,37 @@ void *test_InterlockedCompareExchangePointer_nf(void * volatile *Destination,
// CHECK: ret ptr %[[RESULT:[0-9]+]]
// CHECK: }
+#if defined(__arm__) || defined(__aarch64__)
+void *test_InterlockedCompareExchangePointer_acq(void * volatile *Destination,
+ void *Exchange, void *Comparand) {
+ return _InterlockedCompareExchangePointer_acq(Destination, Exchange, Comparand);
+}
+
+// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedCompareExchangePointer_acq(ptr {{[a-z_ ]*}}%Destination, ptr {{[a-z_ ]*}}%Exchange, ptr {{[a-z_ ]*}}%Comparand){{.*}}{
+// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = ptrtoint ptr %Exchange to [[iPTR]]
+// CHECK-ARM-ARM64: %[[COMPARAND:[0-9]+]] = ptrtoint ptr %Comparand to [[iPTR]]
+// CHECK-ARM-ARM64: %[[XCHG:[0-9]+]] = cmpxchg volatile ptr %[[DEST:.+]], [[iPTR]] %[[COMPARAND:[0-9]+]], [[iPTR]] %[[EXCHANGE:[0-9]+]] acquire acquire, align {{4|8}}
+// CHECK-ARM-ARM64: %[[EXTRACT:[0-9]+]] = extractvalue { [[iPTR]], i1 } %[[XCHG]], 0
+// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXTRACT]] to ptr
+// CHECK-ARM-ARM64: ret ptr %[[RESULT:[0-9]+]]
+// CHECK-ARM-ARM64: }
+
+
+void *test_InterlockedCompareExchangePointer_rel(void * volatile *Destination,
+ void *Exchange, void *Comparand) {
+ return _InterlockedCompareExchangePointer_rel(Destination, Exchange, Comparand);
+}
+
+// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedCompareExchangePointer_rel(ptr {{[a-z_ ]*}}%Destination, ptr {{[a-z_ ]*}}%Exchange, ptr {{[a-z_ ]*}}%Comparand){{.*}}{
+// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = ptrtoint ptr %Exchange to [[iPTR]]
+// CHECK-ARM-ARM64: %[[COMPARAND:[0-9]+]] = ptrtoint ptr %Comparand to [[iPTR]]
+// CHECK-ARM-ARM64: %[[XCHG:[0-9]+]] = cmpxchg volatile ptr %[[DEST:.+]], [[iPTR]] %[[COMPARAND:[0-9]+]], [[iPTR]] %[[EXCHANGE:[0-9]+]] release monotonic, align {{4|8}}
+// CHECK-ARM-ARM64: %[[EXTRACT:[0-9]+]] = extractvalue { [[iPTR]], i1 } %[[XCHG]], 0
+// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXTRACT]] to ptr
+// CHECK-ARM-ARM64: ret ptr %[[RESULT:[0-9]+]]
+// CHECK-ARM-ARM64: }
+#endif
+
char test_InterlockedExchange8(char volatile *value, char mask) {
return _InterlockedExchange8(value, mask);
}
|
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
3 participants
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Adds support for the following MSVC intrinsics:
_InterlockedCompareExchangePointer_acq_InterlockedCompareExchangePointer_rel_InterlockedExchangePointer_acq_InterlockedExchangePointer_nf_InterlockedExchangePointer_relThese are documented at: https://learn.microsoft.com/en-us/cpp/intrinsics/arm64-intrinsics?view=msvc-170#interlocked-intrinsics
NOTE:
_InterlockedCompareExchangePointer_nfis not being added since it already exists, although it was incorrectly added for all architectures instead of being Arm & AArch64 specific.This change also unifies how the pointer and non-pointer interlocked compare-exchange intrinsics are being handled.