Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions clang/include/clang/Basic/BuiltinsAArch64.def
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,9 @@ TARGET_HEADER_BUILTIN(_InterlockedExchange_rel, "NiNiD*Ni", "nh", INTRIN_H,
TARGET_HEADER_BUILTIN(_InterlockedExchange64_acq, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchange64_nf, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchange64_rel, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_acq, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_nf, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_rel, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")

TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_acq, "ccD*cc", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_nf, "ccD*cc", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
Expand All @@ -195,6 +198,8 @@ TARGET_HEADER_BUILTIN(_InterlockedCompareExchange_rel, "NiNiD*NiNi", "nh",
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_acq, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_nf, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_rel, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_acq, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_rel, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")

TARGET_HEADER_BUILTIN(_InterlockedCompareExchange128, "UcLLiD*LLiLLiLLi*", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange128_acq,"UcLLiD*LLiLLiLLi*", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
Expand Down
5 changes: 5 additions & 0 deletions clang/include/clang/Basic/BuiltinsARM.def
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,9 @@ TARGET_HEADER_BUILTIN(_InterlockedExchange_rel, "NiNiD*Ni", "nh", INTRIN_H,
TARGET_HEADER_BUILTIN(_InterlockedExchange64_acq, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchange64_nf, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchange64_rel, "LLiLLiD*LLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_acq, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_nf, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedExchangePointer_rel, "v*v*D*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")

TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_acq, "ccD*cc", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange8_nf, "ccD*cc", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
Expand All @@ -283,6 +286,8 @@ TARGET_HEADER_BUILTIN(_InterlockedCompareExchange_rel, "NiNiD*NiNi", "nh",
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_acq, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_nf, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchange64_rel, "LLiLLiD*LLiLLi", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_acq, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedCompareExchangePointer_rel, "v*v*D*v*v*","nh", INTRIN_H, ALL_MS_LANGUAGES, "")

TARGET_HEADER_BUILTIN(_InterlockedOr8_acq, "ccD*c", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
TARGET_HEADER_BUILTIN(_InterlockedOr8_nf, "ccD*c", "nh", INTRIN_H, ALL_MS_LANGUAGES, "")
Expand Down
65 changes: 35 additions & 30 deletions clang/lib/CodeGen/CGBuiltin.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -509,8 +509,15 @@ Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,

Address DestAddr = CheckAtomicAlignment(CGF, E);

auto *Comparand = CGF.EmitScalarExpr(E->getArg(2));
auto *Exchange = CGF.EmitScalarExpr(E->getArg(1));
auto *RTy = Exchange->getType();

auto *Comparand = CGF.EmitScalarExpr(E->getArg(2));

if (RTy->isPointerTy()) {
Exchange = CGF.Builder.CreatePtrToInt(Exchange, CGF.IntPtrTy);
Comparand = CGF.Builder.CreatePtrToInt(Comparand, CGF.IntPtrTy);
}

// For Release ordering, the failure ordering should be Monotonic.
auto FailureOrdering = SuccessOrdering == AtomicOrdering::Release ?
Expand All @@ -521,10 +528,16 @@ Value *EmitAtomicCmpXchgForMSIntrin(CodeGenFunction &CGF, const CallExpr *E,
// blocks the few atomics optimizations that LLVM has. If we want to optimize
// _Interlocked* operations in the future, we will have to remove the volatile
// marker.
auto *Result = CGF.Builder.CreateAtomicCmpXchg(
auto *CmpXchg = CGF.Builder.CreateAtomicCmpXchg(
DestAddr, Comparand, Exchange, SuccessOrdering, FailureOrdering);
Result->setVolatile(true);
return CGF.Builder.CreateExtractValue(Result, 0);
CmpXchg->setVolatile(true);

auto *Result = CGF.Builder.CreateExtractValue(CmpXchg, 0);
if (RTy->isPointerTy()) {
Result = CGF.Builder.CreateIntToPtr(Result, RTy);
}

return Result;
}

// 64-bit Microsoft platforms support 128 bit cmpxchg operations. They are
Expand Down Expand Up @@ -1595,6 +1608,7 @@ enum class CodeGenFunction::MSVCIntrin {
_BitScanForward,
_BitScanReverse,
_InterlockedAnd,
_InterlockedCompareExchange,
_InterlockedDecrement,
_InterlockedExchange,
_InterlockedExchangeAdd,
Expand Down Expand Up @@ -1680,26 +1694,31 @@ translateArmToMsvcIntrin(unsigned BuiltinID) {
case clang::ARM::BI_InterlockedExchange16_acq:
case clang::ARM::BI_InterlockedExchange_acq:
case clang::ARM::BI_InterlockedExchange64_acq:
case clang::ARM::BI_InterlockedExchangePointer_acq:
return MSVCIntrin::_InterlockedExchange_acq;
case clang::ARM::BI_InterlockedExchange8_rel:
case clang::ARM::BI_InterlockedExchange16_rel:
case clang::ARM::BI_InterlockedExchange_rel:
case clang::ARM::BI_InterlockedExchange64_rel:
case clang::ARM::BI_InterlockedExchangePointer_rel:
return MSVCIntrin::_InterlockedExchange_rel;
case clang::ARM::BI_InterlockedExchange8_nf:
case clang::ARM::BI_InterlockedExchange16_nf:
case clang::ARM::BI_InterlockedExchange_nf:
case clang::ARM::BI_InterlockedExchange64_nf:
case clang::ARM::BI_InterlockedExchangePointer_nf:
return MSVCIntrin::_InterlockedExchange_nf;
case clang::ARM::BI_InterlockedCompareExchange8_acq:
case clang::ARM::BI_InterlockedCompareExchange16_acq:
case clang::ARM::BI_InterlockedCompareExchange_acq:
case clang::ARM::BI_InterlockedCompareExchange64_acq:
case clang::ARM::BI_InterlockedCompareExchangePointer_acq:
return MSVCIntrin::_InterlockedCompareExchange_acq;
case clang::ARM::BI_InterlockedCompareExchange8_rel:
case clang::ARM::BI_InterlockedCompareExchange16_rel:
case clang::ARM::BI_InterlockedCompareExchange_rel:
case clang::ARM::BI_InterlockedCompareExchange64_rel:
case clang::ARM::BI_InterlockedCompareExchangePointer_rel:
return MSVCIntrin::_InterlockedCompareExchange_rel;
case clang::ARM::BI_InterlockedCompareExchange8_nf:
case clang::ARM::BI_InterlockedCompareExchange16_nf:
Expand Down Expand Up @@ -1826,26 +1845,31 @@ translateAarch64ToMsvcIntrin(unsigned BuiltinID) {
case clang::AArch64::BI_InterlockedExchange16_acq:
case clang::AArch64::BI_InterlockedExchange_acq:
case clang::AArch64::BI_InterlockedExchange64_acq:
case clang::AArch64::BI_InterlockedExchangePointer_acq:
return MSVCIntrin::_InterlockedExchange_acq;
case clang::AArch64::BI_InterlockedExchange8_rel:
case clang::AArch64::BI_InterlockedExchange16_rel:
case clang::AArch64::BI_InterlockedExchange_rel:
case clang::AArch64::BI_InterlockedExchange64_rel:
case clang::AArch64::BI_InterlockedExchangePointer_rel:
return MSVCIntrin::_InterlockedExchange_rel;
case clang::AArch64::BI_InterlockedExchange8_nf:
case clang::AArch64::BI_InterlockedExchange16_nf:
case clang::AArch64::BI_InterlockedExchange_nf:
case clang::AArch64::BI_InterlockedExchange64_nf:
case clang::AArch64::BI_InterlockedExchangePointer_nf:
return MSVCIntrin::_InterlockedExchange_nf;
case clang::AArch64::BI_InterlockedCompareExchange8_acq:
case clang::AArch64::BI_InterlockedCompareExchange16_acq:
case clang::AArch64::BI_InterlockedCompareExchange_acq:
case clang::AArch64::BI_InterlockedCompareExchange64_acq:
case clang::AArch64::BI_InterlockedCompareExchangePointer_acq:
return MSVCIntrin::_InterlockedCompareExchange_acq;
case clang::AArch64::BI_InterlockedCompareExchange8_rel:
case clang::AArch64::BI_InterlockedCompareExchange16_rel:
case clang::AArch64::BI_InterlockedCompareExchange_rel:
case clang::AArch64::BI_InterlockedCompareExchange64_rel:
case clang::AArch64::BI_InterlockedCompareExchangePointer_rel:
return MSVCIntrin::_InterlockedCompareExchange_rel;
case clang::AArch64::BI_InterlockedCompareExchange8_nf:
case clang::AArch64::BI_InterlockedCompareExchange16_nf:
Expand Down Expand Up @@ -2048,6 +2072,8 @@ Value *CodeGenFunction::EmitMSVCBuiltinExpr(MSVCIntrin BuiltinID,
case MSVCIntrin::_InterlockedExchange_nf:
return MakeBinaryAtomicValue(*this, AtomicRMWInst::Xchg, E,
AtomicOrdering::Monotonic);
case MSVCIntrin::_InterlockedCompareExchange:
return EmitAtomicCmpXchgForMSIntrin(*this, E);
case MSVCIntrin::_InterlockedCompareExchange_acq:
return EmitAtomicCmpXchgForMSIntrin(*this, E, AtomicOrdering::Acquire);
case MSVCIntrin::_InterlockedCompareExchange_rel:
Expand Down Expand Up @@ -5695,32 +5721,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
return RValue::get(
EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedExchange, E));
case Builtin::BI_InterlockedCompareExchangePointer:
case Builtin::BI_InterlockedCompareExchangePointer_nf: {
llvm::Type *RTy;
llvm::IntegerType *IntType = IntegerType::get(
getLLVMContext(), getContext().getTypeSize(E->getType()));

Address DestAddr = CheckAtomicAlignment(*this, E);

llvm::Value *Exchange = EmitScalarExpr(E->getArg(1));
RTy = Exchange->getType();
Exchange = Builder.CreatePtrToInt(Exchange, IntType);

llvm::Value *Comparand =
Builder.CreatePtrToInt(EmitScalarExpr(E->getArg(2)), IntType);

auto Ordering =
BuiltinID == Builtin::BI_InterlockedCompareExchangePointer_nf ?
AtomicOrdering::Monotonic : AtomicOrdering::SequentiallyConsistent;

auto Result = Builder.CreateAtomicCmpXchg(DestAddr, Comparand, Exchange,
Ordering, Ordering);
Result->setVolatile(true);

return RValue::get(Builder.CreateIntToPtr(Builder.CreateExtractValue(Result,
0),
RTy));
}
return RValue::get(
EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange, E));
case Builtin::BI_InterlockedCompareExchangePointer_nf:
return RValue::get(
EmitMSVCBuiltinExpr(MSVCIntrin::_InterlockedCompareExchange_nf, E));
case Builtin::BI_InterlockedCompareExchange8:
case Builtin::BI_InterlockedCompareExchange16:
case Builtin::BI_InterlockedCompareExchange:
Expand Down
9 changes: 9 additions & 0 deletions clang/lib/Headers/intrin0.h
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,9 @@ long _InterlockedExchange_rel(long volatile *_Target, long _Value);
__int64 _InterlockedExchange64_acq(__int64 volatile *_Target, __int64 _Value);
__int64 _InterlockedExchange64_nf(__int64 volatile *_Target, __int64 _Value);
__int64 _InterlockedExchange64_rel(__int64 volatile *_Target, __int64 _Value);
void *_InterlockedExchangePointer_acq(void *volatile *_Target, void *_Value);
void *_InterlockedExchangePointer_nf(void *volatile *_Target, void *_Value);
void *_InterlockedExchangePointer_rel(void *volatile *_Target, void *_Value);

/*----------------------------------------------------------------------------*\
|* Interlocked Compare Exchange
Expand Down Expand Up @@ -237,6 +240,12 @@ __int64 _InterlockedCompareExchange64_nf(__int64 volatile *_Destination,
__int64 _InterlockedCompareExchange64_rel(__int64 volatile *_Destination,
__int64 _Exchange,
__int64 _Comparand);
void *_InterlockedCompareExchangePointer_acq(void *volatile *_Destination,
void *_Exchange, void *_Comparand);
void *_InterlockedCompareExchangePointer_nf(void *volatile *_Destination,
void *_Exchange, void *_Comparand);
void *_InterlockedCompareExchangePointer_rel(void *volatile *_Destination,
void *_Exchange, void *_Comparand);
#endif

#ifdef __cplusplus
Expand Down
66 changes: 66 additions & 0 deletions clang/test/CodeGen/ms-intrinsics.c
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,41 @@ void *test_InterlockedExchangePointer(void * volatile *Target, void *Value) {
// CHECK: ret ptr %[[RESULT]]
// CHECK: }

#if defined(__arm__) || defined(__aarch64__)
void *test_InterlockedExchangePointer_acq(void * volatile *Target, void *Value) {
return _InterlockedExchangePointer_acq(Target, Value);
}

// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedExchangePointer_acq(ptr {{[a-z_ ]*}}%Target, ptr {{[a-z_ ]*}}%Value){{.*}}{
// CHECK-ARM-ARM64: %[[VALUE:[0-9]+]] = ptrtoint ptr %Value to [[iPTR:i[0-9]+]]
// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = atomicrmw xchg ptr %Target, [[iPTR]] %[[VALUE]] acquire, align {{4|8}}
// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXCHANGE]] to ptr
// CHECK-ARM-ARM64: ret ptr %[[RESULT]]
// CHECK-ARM-ARM64: }

void *test_InterlockedExchangePointer_nf(void * volatile *Target, void *Value) {
return _InterlockedExchangePointer_nf(Target, Value);
}

// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedExchangePointer_nf(ptr {{[a-z_ ]*}}%Target, ptr {{[a-z_ ]*}}%Value){{.*}}{
// CHECK-ARM-ARM64: %[[VALUE:[0-9]+]] = ptrtoint ptr %Value to [[iPTR]]
// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = atomicrmw xchg ptr %Target, [[iPTR]] %[[VALUE]] monotonic, align {{4|8}}
// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXCHANGE]] to ptr
// CHECK-ARM-ARM64: ret ptr %[[RESULT]]
// CHECK-ARM-ARM64: }

void *test_InterlockedExchangePointer_rel(void * volatile *Target, void *Value) {
return _InterlockedExchangePointer_rel(Target, Value);
}

// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedExchangePointer_rel(ptr {{[a-z_ ]*}}%Target, ptr {{[a-z_ ]*}}%Value){{.*}}{
// CHECK-ARM-ARM64: %[[VALUE:[0-9]+]] = ptrtoint ptr %Value to [[iPTR]]
// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = atomicrmw xchg ptr %Target, [[iPTR]] %[[VALUE]] release, align {{4|8}}
// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXCHANGE]] to ptr
// CHECK-ARM-ARM64: ret ptr %[[RESULT]]
// CHECK-ARM-ARM64: }
#endif

void *test_InterlockedCompareExchangePointer(void * volatile *Destination,
void *Exchange, void *Comparand) {
return _InterlockedCompareExchangePointer(Destination, Exchange, Comparand);
Expand Down Expand Up @@ -249,6 +284,37 @@ void *test_InterlockedCompareExchangePointer_nf(void * volatile *Destination,
// CHECK: ret ptr %[[RESULT:[0-9]+]]
// CHECK: }

#if defined(__arm__) || defined(__aarch64__)
void *test_InterlockedCompareExchangePointer_acq(void * volatile *Destination,
void *Exchange, void *Comparand) {
return _InterlockedCompareExchangePointer_acq(Destination, Exchange, Comparand);
}

// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedCompareExchangePointer_acq(ptr {{[a-z_ ]*}}%Destination, ptr {{[a-z_ ]*}}%Exchange, ptr {{[a-z_ ]*}}%Comparand){{.*}}{
// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = ptrtoint ptr %Exchange to [[iPTR]]
// CHECK-ARM-ARM64: %[[COMPARAND:[0-9]+]] = ptrtoint ptr %Comparand to [[iPTR]]
// CHECK-ARM-ARM64: %[[XCHG:[0-9]+]] = cmpxchg volatile ptr %[[DEST:.+]], [[iPTR]] %[[COMPARAND:[0-9]+]], [[iPTR]] %[[EXCHANGE:[0-9]+]] acquire acquire, align {{4|8}}
// CHECK-ARM-ARM64: %[[EXTRACT:[0-9]+]] = extractvalue { [[iPTR]], i1 } %[[XCHG]], 0
// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXTRACT]] to ptr
// CHECK-ARM-ARM64: ret ptr %[[RESULT:[0-9]+]]
// CHECK-ARM-ARM64: }


void *test_InterlockedCompareExchangePointer_rel(void * volatile *Destination,
void *Exchange, void *Comparand) {
return _InterlockedCompareExchangePointer_rel(Destination, Exchange, Comparand);
}

// CHECK-ARM-ARM64: define{{.*}}ptr @test_InterlockedCompareExchangePointer_rel(ptr {{[a-z_ ]*}}%Destination, ptr {{[a-z_ ]*}}%Exchange, ptr {{[a-z_ ]*}}%Comparand){{.*}}{
// CHECK-ARM-ARM64: %[[EXCHANGE:[0-9]+]] = ptrtoint ptr %Exchange to [[iPTR]]
// CHECK-ARM-ARM64: %[[COMPARAND:[0-9]+]] = ptrtoint ptr %Comparand to [[iPTR]]
// CHECK-ARM-ARM64: %[[XCHG:[0-9]+]] = cmpxchg volatile ptr %[[DEST:.+]], [[iPTR]] %[[COMPARAND:[0-9]+]], [[iPTR]] %[[EXCHANGE:[0-9]+]] release monotonic, align {{4|8}}
// CHECK-ARM-ARM64: %[[EXTRACT:[0-9]+]] = extractvalue { [[iPTR]], i1 } %[[XCHG]], 0
// CHECK-ARM-ARM64: %[[RESULT:[0-9]+]] = inttoptr [[iPTR]] %[[EXTRACT]] to ptr
// CHECK-ARM-ARM64: ret ptr %[[RESULT:[0-9]+]]
// CHECK-ARM-ARM64: }
#endif

char test_InterlockedExchange8(char volatile *value, char mask) {
return _InterlockedExchange8(value, mask);
}
Expand Down
Loading