@@ -374,11 +374,9 @@ bool AtomicInfo::emitMemSetZeroIfNecessary() const {
374374}
375375
376376static void emitAtomicCmpXchg (CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
377- Address Dest, Address Ptr,
378- Address Val1, Address Val2,
379- Address ExpectedResult,
380- uint64_t Size,
381- llvm::AtomicOrdering SuccessOrder,
377+ Address Dest, Address Ptr, Address Val1,
378+ Address Val2, Address ExpectedResult,
379+ uint64_t Size, llvm::AtomicOrdering SuccessOrder,
382380 llvm::AtomicOrdering FailureOrder,
383381 llvm::SyncScope::ID Scope) {
384382 // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
@@ -412,34 +410,31 @@ static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
412410
413411 CGF.Builder .SetInsertPoint (StoreExpectedBB);
414412 // Update the memory at Expected with Old's value.
415- llvm::Type *ExpectedType = ExpectedResult.getElementType ();
416- const llvm::DataLayout &DL = CGF.CGM .getDataLayout ();
417- uint64_t ExpectedSizeInBytes = DL.getTypeStoreSize (ExpectedType);
418-
419- if (ExpectedSizeInBytes == Size) {
420- // Sizes match: store directly
421- auto I* = CGF.Builder .CreateStore (Old, ExpectedResult);
422- CGF.addInstToCurrentSourceAtom (I, Old);
423- } else {
424- // store only the first ExpectedSizeInBytes bytes of Old
425- llvm::Type *OldType = Old->getType ();
426-
427- // Allocate temporary storage for Old value
428- Address OldTmp = CGF.CreateTempAlloca (OldType, Ptr.getAlignment (), " old.tmp" );
429-
430- // Store Old into this temporary
431- auto *I = CGF.Builder .CreateStore (Old, OldTmp);
432- CGF.addInstToCurrentSourceAtom (I, Old);
433-
434- // Perform memcpy for first ExpectedSizeInBytes bytes
435- CGF.Builder .CreateMemCpy (
436- ExpectedResult,
437- OldTmp,
438- ExpectedSizeInBytes,
439- /* isVolatile=*/ false );
440- }
413+ llvm::Type *ExpectedType = ExpectedResult.getElementType ();
414+ const llvm::DataLayout &DL = CGF.CGM .getDataLayout ();
415+ uint64_t ExpectedSizeInBytes = DL.getTypeStoreSize (ExpectedType);
416+
417+ if (ExpectedSizeInBytes == Size) {
418+ // Sizes match: store directly
419+ auto *I = CGF.Builder .CreateStore (Old, ExpectedResult);
420+ CGF.addInstToCurrentSourceAtom (I, Old);
421+ } else {
422+ // store only the first ExpectedSizeInBytes bytes of Old
423+ llvm::Type *OldType = Old->getType ();
424+
425+ // Allocate temporary storage for Old value
426+ Address OldTmp =
427+ CGF.CreateTempAlloca (OldType, Ptr.getAlignment (), " old.tmp" );
428+
429+ // Store Old into this temporary
430+ auto *I = CGF.Builder .CreateStore (Old, OldTmp);
431+ CGF.addInstToCurrentSourceAtom (I, Old);
432+
433+ // Perform memcpy for first ExpectedSizeInBytes bytes
434+ CGF.Builder .CreateMemCpy (ExpectedResult, OldTmp, ExpectedSizeInBytes,
435+ /* isVolatile=*/ false );
436+ }
441437
442-
443438 // Finally, branch to the exit point.
444439 CGF.Builder .CreateBr (ContinueBB);
445440
@@ -451,14 +446,11 @@ if (ExpectedSizeInBytes == Size) {
451446// / Given an ordering required on success, emit all possible cmpxchg
452447// / instructions to cope with the provided (but possibly only dynamically known)
453448// / FailureOrder.
454- static void emitAtomicCmpXchgFailureSet (CodeGenFunction &CGF, AtomicExpr *E,
455- bool IsWeak, Address Dest, Address Ptr,
456- Address Val1, Address Val2,
457- Address ExpectedResult,
458- llvm::Value *FailureOrderVal,
459- uint64_t Size,
460- llvm::AtomicOrdering SuccessOrder,
461- llvm::SyncScope::ID Scope) {
449+ static void emitAtomicCmpXchgFailureSet (
450+ CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, Address Dest, Address Ptr,
451+ Address Val1, Address Val2, Address ExpectedResult,
452+ llvm::Value *FailureOrderVal, uint64_t Size,
453+ llvm::AtomicOrdering SuccessOrder, llvm::SyncScope::ID Scope) {
462454 llvm::AtomicOrdering FailureOrder;
463455 if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
464456 auto FOS = FO->getSExtValue ();
@@ -485,8 +477,8 @@ static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
485477 // success argument". This condition has been lifted and the only
486478 // precondition is 31.7.2.18. Effectively treat this as a DR and skip
487479 // language version checks.
488- emitAtomicCmpXchg (CGF, E, IsWeak, Dest, Ptr, Val1, Val2, ExpectedResult, Size, SuccessOrder,
489- FailureOrder, Scope);
480+ emitAtomicCmpXchg (CGF, E, IsWeak, Dest, Ptr, Val1, Val2, ExpectedResult,
481+ Size, SuccessOrder, FailureOrder, Scope);
490482 return ;
491483 }
492484
@@ -510,18 +502,19 @@ static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
510502
511503 // Emit all the different atomics
512504 CGF.Builder .SetInsertPoint (MonotonicBB);
513- emitAtomicCmpXchg (CGF, E, IsWeak, Dest, Ptr, Val1, Val2, ExpectedResult,
514- Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope);
505+ emitAtomicCmpXchg (CGF, E, IsWeak, Dest, Ptr, Val1, Val2, ExpectedResult, Size,
506+ SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope);
515507 CGF.Builder .CreateBr (ContBB);
516508
517509 CGF.Builder .SetInsertPoint (AcquireBB);
518- emitAtomicCmpXchg (CGF, E, IsWeak, Dest, Ptr, Val1, Val2, ExpectedResult, Size, SuccessOrder,
519- llvm::AtomicOrdering::Acquire, Scope);
510+ emitAtomicCmpXchg (CGF, E, IsWeak, Dest, Ptr, Val1, Val2, ExpectedResult, Size,
511+ SuccessOrder, llvm::AtomicOrdering::Acquire, Scope);
520512 CGF.Builder .CreateBr (ContBB);
521513
522514 CGF.Builder .SetInsertPoint (SeqCstBB);
523- emitAtomicCmpXchg (CGF, E, IsWeak, Dest, Ptr, Val1, Val2, ExpectedResult, Size, SuccessOrder,
524- llvm::AtomicOrdering::SequentiallyConsistent, Scope);
515+ emitAtomicCmpXchg (CGF, E, IsWeak, Dest, Ptr, Val1, Val2, ExpectedResult, Size,
516+ SuccessOrder, llvm::AtomicOrdering::SequentiallyConsistent,
517+ Scope);
525518 CGF.Builder .CreateBr (ContBB);
526519
527520 CGF.Builder .SetInsertPoint (ContBB);
@@ -553,9 +546,9 @@ static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder,
553546
554547static void EmitAtomicOp (CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
555548 Address Ptr, Address Val1, Address Val2,
556- Address ExpectedResult,
557- llvm::Value *IsWeak, llvm::Value *FailureOrder ,
558- uint64_t Size, llvm::AtomicOrdering Order,
549+ Address ExpectedResult, llvm::Value *IsWeak,
550+ llvm::Value *FailureOrder, uint64_t Size ,
551+ llvm::AtomicOrdering Order,
559552 llvm::SyncScope::ID Scope) {
560553 llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
561554 bool PostOpMinMax = false ;
@@ -570,21 +563,24 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
570563 case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
571564 case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
572565 emitAtomicCmpXchgFailureSet (CGF, E, false , Dest, Ptr, Val1, Val2,
573- ExpectedResult, FailureOrder, Size, Order, Scope);
566+ ExpectedResult, FailureOrder, Size, Order,
567+ Scope);
574568 return ;
575569 case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
576570 case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
577571 case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
578572 emitAtomicCmpXchgFailureSet (CGF, E, true , Dest, Ptr, Val1, Val2,
579- ExpectedResult, FailureOrder, Size, Order, Scope);
573+ ExpectedResult, FailureOrder, Size, Order,
574+ Scope);
580575 return ;
581576 case AtomicExpr::AO__atomic_compare_exchange:
582577 case AtomicExpr::AO__atomic_compare_exchange_n:
583578 case AtomicExpr::AO__scoped_atomic_compare_exchange:
584579 case AtomicExpr::AO__scoped_atomic_compare_exchange_n: {
585580 if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
586581 emitAtomicCmpXchgFailureSet (CGF, E, IsWeakC->getZExtValue (), Dest, Ptr,
587- Val1, Val2, ExpectedResult, FailureOrder, Size, Order, Scope);
582+ Val1, Val2, ExpectedResult, FailureOrder,
583+ Size, Order, Scope);
588584 } else {
589585 // Create all the relevant BB's
590586 llvm::BasicBlock *StrongBB =
@@ -598,12 +594,14 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
598594
599595 CGF.Builder .SetInsertPoint (StrongBB);
600596 emitAtomicCmpXchgFailureSet (CGF, E, false , Dest, Ptr, Val1, Val2,
601- ExpectedResult, FailureOrder, Size, Order, Scope);
597+ ExpectedResult, FailureOrder, Size, Order,
598+ Scope);
602599 CGF.Builder .CreateBr (ContBB);
603600
604601 CGF.Builder .SetInsertPoint (WeakBB);
605602 emitAtomicCmpXchgFailureSet (CGF, E, true , Dest, Ptr, Val1, Val2,
606- ExpectedResult, FailureOrder, Size, Order, Scope);
603+ ExpectedResult, FailureOrder, Size, Order,
604+ Scope);
607605 CGF.Builder .CreateBr (ContBB);
608606
609607 CGF.Builder .SetInsertPoint (ContBB);
@@ -813,10 +811,9 @@ EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
813811
814812static void EmitAtomicOp (CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,
815813 Address Ptr, Address Val1, Address Val2,
816- Address OriginalVal1,
817- llvm::Value *IsWeak, llvm::Value *FailureOrder,
818- uint64_t Size, llvm::AtomicOrdering Order,
819- llvm::Value *Scope) {
814+ Address OriginalVal1, llvm::Value *IsWeak,
815+ llvm::Value *FailureOrder, uint64_t Size,
816+ llvm::AtomicOrdering Order, llvm::Value *Scope) {
820817 auto ScopeModel = Expr->getScopeModel ();
821818
822819 // LLVM atomic instructions always have sync scope. If clang atomic
@@ -833,8 +830,8 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,
833830 Order, CGF.getLLVMContext ());
834831 else
835832 SS = llvm::SyncScope::System;
836- EmitAtomicOp (CGF, Expr, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak, FailureOrder, Size,
837- Order, SS);
833+ EmitAtomicOp (CGF, Expr, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,
834+ FailureOrder, Size, Order, SS);
838835 return ;
839836 }
840837
@@ -843,8 +840,8 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,
843840 auto SCID = CGF.getTargetHooks ().getLLVMSyncScopeID (
844841 CGF.CGM .getLangOpts (), ScopeModel->map (SC->getZExtValue ()),
845842 Order, CGF.CGM .getLLVMContext ());
846- EmitAtomicOp (CGF, Expr, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak, FailureOrder, Size,
847- Order, SCID);
843+ EmitAtomicOp (CGF, Expr, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,
844+ FailureOrder, Size, Order, SCID);
848845 return ;
849846 }
850847
@@ -869,12 +866,11 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,
869866 SI->addCase (Builder.getInt32 (S), B);
870867
871868 Builder.SetInsertPoint (B);
872- EmitAtomicOp (CGF, Expr, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak, FailureOrder, Size,
873- Order,
874- CGF.getTargetHooks ().getLLVMSyncScopeID (CGF.CGM .getLangOpts (),
875- ScopeModel->map (S),
876- Order,
877- CGF.getLLVMContext ()));
869+ EmitAtomicOp (CGF, Expr, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,
870+ FailureOrder, Size, Order,
871+ CGF.getTargetHooks ().getLLVMSyncScopeID (
872+ CGF.CGM .getLangOpts (), ScopeModel->map (S), Order,
873+ CGF.getLLVMContext ()));
878874 Builder.CreateBr (ContBB);
879875 }
880876
@@ -1319,30 +1315,32 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
13191315 if (llvm::isValidAtomicOrderingCABI (ord))
13201316 switch ((llvm::AtomicOrderingCABI)ord) {
13211317 case llvm::AtomicOrderingCABI::relaxed:
1322- EmitAtomicOp (*this , E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak, OrderFail, Size,
1323- llvm::AtomicOrdering::Monotonic, Scope);
1318+ EmitAtomicOp (*this , E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,
1319+ OrderFail, Size, llvm::AtomicOrdering::Monotonic, Scope);
13241320 break ;
13251321 case llvm::AtomicOrderingCABI::consume:
13261322 case llvm::AtomicOrderingCABI::acquire:
13271323 if (IsStore)
13281324 break ; // Avoid crashing on code with undefined behavior
1329- EmitAtomicOp (*this , E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak, OrderFail, Size,
1330- llvm::AtomicOrdering::Acquire, Scope);
1325+ EmitAtomicOp (*this , E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,
1326+ OrderFail, Size, llvm::AtomicOrdering::Acquire, Scope);
13311327 break ;
13321328 case llvm::AtomicOrderingCABI::release:
13331329 if (IsLoad)
13341330 break ; // Avoid crashing on code with undefined behavior
1335- EmitAtomicOp (*this , E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak, OrderFail, Size,
1336- llvm::AtomicOrdering::Release, Scope);
1331+ EmitAtomicOp (*this , E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,
1332+ OrderFail, Size, llvm::AtomicOrdering::Release, Scope);
13371333 break ;
13381334 case llvm::AtomicOrderingCABI::acq_rel:
13391335 if (IsLoad || IsStore)
13401336 break ; // Avoid crashing on code with undefined behavior
1341- EmitAtomicOp (*this , E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak, OrderFail, Size,
1342- llvm::AtomicOrdering::AcquireRelease, Scope);
1337+ EmitAtomicOp (*this , E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,
1338+ OrderFail, Size, llvm::AtomicOrdering::AcquireRelease,
1339+ Scope);
13431340 break ;
13441341 case llvm::AtomicOrderingCABI::seq_cst:
1345- EmitAtomicOp (*this , E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak, OrderFail, Size,
1342+ EmitAtomicOp (*this , E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,
1343+ OrderFail, Size,
13461344 llvm::AtomicOrdering::SequentiallyConsistent, Scope);
13471345 break ;
13481346 }
@@ -1378,13 +1376,13 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
13781376
13791377 // Emit all the different atomics
13801378 Builder.SetInsertPoint (MonotonicBB);
1381- EmitAtomicOp (*this , E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak, OrderFail, Size,
1382- llvm::AtomicOrdering::Monotonic, Scope);
1379+ EmitAtomicOp (*this , E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak, OrderFail,
1380+ Size, llvm::AtomicOrdering::Monotonic, Scope);
13831381 Builder.CreateBr (ContBB);
13841382 if (!IsStore) {
13851383 Builder.SetInsertPoint (AcquireBB);
1386- EmitAtomicOp (*this , E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak, OrderFail, Size,
1387- llvm::AtomicOrdering::Acquire, Scope);
1384+ EmitAtomicOp (*this , E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,
1385+ OrderFail, Size, llvm::AtomicOrdering::Acquire, Scope);
13881386 Builder.CreateBr (ContBB);
13891387 SI->addCase (Builder.getInt32 ((int )llvm::AtomicOrderingCABI::consume),
13901388 AcquireBB);
@@ -1393,23 +1391,23 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
13931391 }
13941392 if (!IsLoad) {
13951393 Builder.SetInsertPoint (ReleaseBB);
1396- EmitAtomicOp (*this , E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak, OrderFail, Size,
1397- llvm::AtomicOrdering::Release, Scope);
1394+ EmitAtomicOp (*this , E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,
1395+ OrderFail, Size, llvm::AtomicOrdering::Release, Scope);
13981396 Builder.CreateBr (ContBB);
13991397 SI->addCase (Builder.getInt32 ((int )llvm::AtomicOrderingCABI::release),
14001398 ReleaseBB);
14011399 }
14021400 if (!IsLoad && !IsStore) {
14031401 Builder.SetInsertPoint (AcqRelBB);
1404- EmitAtomicOp (*this , E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak, OrderFail, Size,
1405- llvm::AtomicOrdering::AcquireRelease, Scope);
1402+ EmitAtomicOp (*this , E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak,
1403+ OrderFail, Size, llvm::AtomicOrdering::AcquireRelease, Scope);
14061404 Builder.CreateBr (ContBB);
14071405 SI->addCase (Builder.getInt32 ((int )llvm::AtomicOrderingCABI::acq_rel),
14081406 AcqRelBB);
14091407 }
14101408 Builder.SetInsertPoint (SeqCstBB);
1411- EmitAtomicOp (*this , E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak, OrderFail, Size,
1412- llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1409+ EmitAtomicOp (*this , E, Dest, Ptr, Val1, Val2, OriginalVal1, IsWeak, OrderFail,
1410+ Size, llvm::AtomicOrdering::SequentiallyConsistent, Scope);
14131411 Builder.CreateBr (ContBB);
14141412 SI->addCase (Builder.getInt32 ((int )llvm::AtomicOrderingCABI::seq_cst),
14151413 SeqCstBB);
0 commit comments