From 96cb280f4b687e57bc362bbb250f9208394aba5f Mon Sep 17 00:00:00 2001 From: Maryam Moghadas Date: Fri, 21 Nov 2025 20:45:26 +0000 Subject: [PATCH] [PowerPC] Add AMO load builtins for conditional increment/decrement --- clang/include/clang/Basic/BuiltinsPPC.def | 4 + clang/lib/CodeGen/TargetBuiltins/PPC.cpp | 12 +++ clang/lib/Sema/SemaPPC.cpp | 19 +++++ clang/test/CodeGen/PowerPC/builtins-amo-err.c | 24 ++++++ clang/test/CodeGen/PowerPC/builtins-ppc-amo.c | 77 ++++++++++++++++++- llvm/include/llvm/IR/IntrinsicsPowerPC.td | 8 ++ llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 43 ++++++++++- llvm/lib/Target/PowerPC/PPCInstr64Bit.td | 6 ++ llvm/lib/Target/PowerPC/PPCInstrInfo.td | 6 ++ llvm/test/CodeGen/PowerPC/amo-enable.ll | 38 +++++++++ 10 files changed, 235 insertions(+), 2 deletions(-) diff --git a/clang/include/clang/Basic/BuiltinsPPC.def b/clang/include/clang/Basic/BuiltinsPPC.def index 43ba793795d38..7689daf00e6a7 100644 --- a/clang/include/clang/Basic/BuiltinsPPC.def +++ b/clang/include/clang/Basic/BuiltinsPPC.def @@ -1006,6 +1006,10 @@ TARGET_BUILTIN(__builtin_amo_lwat, "UiUi*UiIi", "", "isa-v30-instructions") TARGET_BUILTIN(__builtin_amo_ldat, "ULiULi*ULiIi", "", "isa-v30-instructions") TARGET_BUILTIN(__builtin_amo_lwat_s, "SiSi*SiIi", "", "isa-v30-instructions") TARGET_BUILTIN(__builtin_amo_ldat_s, "SLiSLi*SLiIi", "", "isa-v30-instructions") +TARGET_BUILTIN(__builtin_amo_lwat_cond, "UiUi*Ii", "", "isa-v30-instructions") +TARGET_BUILTIN(__builtin_amo_ldat_cond, "ULiULi*Ii", "", "isa-v30-instructions") +TARGET_BUILTIN(__builtin_amo_lwat_cond_s, "SiSi*Ii", "", "isa-v30-instructions") +TARGET_BUILTIN(__builtin_amo_ldat_cond_s, "SLiSLi*Ii", "", "isa-v30-instructions") // Set the floating point rounding mode BUILTIN(__builtin_setrnd, "di", "") diff --git a/clang/lib/CodeGen/TargetBuiltins/PPC.cpp b/clang/lib/CodeGen/TargetBuiltins/PPC.cpp index 1e1127350c301..bccb6acdb4e06 100644 --- a/clang/lib/CodeGen/TargetBuiltins/PPC.cpp +++ b/clang/lib/CodeGen/TargetBuiltins/PPC.cpp @@ -1374,5 +1374,17 @@ Value *CodeGenFunction::EmitPPCBuiltinExpr(unsigned BuiltinID, return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_amo_ldat), {Op0, Op1, Op2}); } + case PPC::BI__builtin_amo_lwat_cond_s: { + Value *Op0 = EmitScalarExpr(E->getArg(0)); + Value *Op1 = EmitScalarExpr(E->getArg(1)); + return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_amo_lwat_cond), + {Op0, Op1}); + } + case PPC::BI__builtin_amo_ldat_cond_s: { + Value *Op0 = EmitScalarExpr(E->getArg(0)); + Value *Op1 = EmitScalarExpr(E->getArg(1)); + return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::ppc_amo_ldat_cond), + {Op0, Op1}); + } } } diff --git a/clang/lib/Sema/SemaPPC.cpp b/clang/lib/Sema/SemaPPC.cpp index 536ba11088c8d..a7e76a9917372 100644 --- a/clang/lib/Sema/SemaPPC.cpp +++ b/clang/lib/Sema/SemaPPC.cpp @@ -91,6 +91,10 @@ static bool isPPC_64Builtin(unsigned BuiltinID) { case PPC::BI__builtin_amo_ldat: case PPC::BI__builtin_amo_lwat_s: case PPC::BI__builtin_amo_ldat_s: + case PPC::BI__builtin_amo_lwat_cond: + case PPC::BI__builtin_amo_ldat_cond: + case PPC::BI__builtin_amo_lwat_cond_s: + case PPC::BI__builtin_amo_ldat_cond_s: return true; } return false; @@ -281,6 +285,21 @@ bool SemaPPC::CheckPPCBuiltinFunctionCall(const TargetInfo &TI, << toString(Result, 10) << (IsUnsigned ? "0-4, 6" : "0, 5, 7") << "8" << Arg->getSourceRange(); } + case PPC::BI__builtin_amo_lwat_cond: + case PPC::BI__builtin_amo_ldat_cond: + case PPC::BI__builtin_amo_lwat_cond_s: + case PPC::BI__builtin_amo_ldat_cond_s: { + llvm::APSInt Result; + if (SemaRef.BuiltinConstantArg(TheCall, 1, Result)) + return true; + unsigned Val = Result.getZExtValue(); + if (llvm::is_contained({24u, 25u, 28u}, Val)) + return false; + + Expr *Arg = TheCall->getArg(1); + return SemaRef.Diag(Arg->getBeginLoc(), diag::err_argument_invalid_range) + << toString(Result, 10) << "24, 25" << "28" << Arg->getSourceRange(); + } } llvm_unreachable("must return from switch"); } diff --git a/clang/test/CodeGen/PowerPC/builtins-amo-err.c b/clang/test/CodeGen/PowerPC/builtins-amo-err.c index 5dc6445c6ec5a..ad6be9e867856 100644 --- a/clang/test/CodeGen/PowerPC/builtins-amo-err.c +++ b/clang/test/CodeGen/PowerPC/builtins-amo-err.c @@ -27,4 +27,28 @@ void test_amo() { __builtin_amo_ldat_s(ptr4, value4, 5); // FC-ERROR: error: argument value 6 is outside the valid range [0, 5, 7, 8] __builtin_amo_ldat_s(ptr4, value4, 6); + + unsigned int *ptr5; + // AIX32-ERROR-COUNT-2: error: this builtin is only available on 64-bit targets + __builtin_amo_lwat_cond(ptr5, 24); + // FC-ERROR: argument value 20 is outside the valid range [24, 25, 28] + __builtin_amo_lwat_cond(ptr5, 20); + + unsigned long int *ptr6; + // AIX32-ERROR-COUNT-2: error: this builtin is only available on 64-bit targets + __builtin_amo_ldat_cond(ptr6, 28); + // FC-ERROR: argument value 0 is outside the valid range [24, 25, 28] + __builtin_amo_ldat_cond(ptr6, 0); + + signed int *ptr7; + // AIX32-ERROR-COUNT-2: error: this builtin is only available on 64-bit targets + __builtin_amo_lwat_cond_s(ptr7, 24); + // FC-ERROR: argument value 20 is outside the valid range [24, 25, 28] + __builtin_amo_lwat_cond_s(ptr7, 20); + + signed long int *ptr8; + // AIX32-ERROR-COUNT-2: error: this builtin is only available on 64-bit targets + __builtin_amo_ldat_cond_s(ptr6, 28); + // FC-ERROR: argument value 0 is outside the valid range [24, 25, 28] + __builtin_amo_ldat_cond_s(ptr6, 0); } diff --git a/clang/test/CodeGen/PowerPC/builtins-ppc-amo.c b/clang/test/CodeGen/PowerPC/builtins-ppc-amo.c index 03e71429319b3..0bf28e85730e1 100644 --- a/clang/test/CodeGen/PowerPC/builtins-ppc-amo.c +++ b/clang/test/CodeGen/PowerPC/builtins-ppc-amo.c @@ -61,7 +61,6 @@ void test_signed_lwat(int *ptr, int value, int * resp) { *resp = res; } - // CHECK-LABEL: define dso_local void @test_signed_ldat( // CHECK-SAME: ptr noundef [[PTR:%.*]], i64 noundef [[VALUE:%.*]], ptr noundef writeonly captures(none) initializes((0, 8)) [[RESP:%.*]]) local_unnamed_addr #[[ATTR0]] { // CHECK-NEXT: [[ENTRY:.*:]] @@ -81,6 +80,82 @@ void test_signed_ldat(long int *ptr, long int value, long int * resp) { *resp = res; } +// CHECK-LABEL: define dso_local void @test_unsigned_lwat_cond( +// CHECK-SAME: ptr noundef [[PTR:%.*]], ptr noundef writeonly captures(none) initializes((0, 4)) [[RESP:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.ppc.amo.lwat.cond(ptr [[PTR]], i32 24) +// CHECK-NEXT: store i32 [[TMP0]], ptr [[RESP]], align 4, !tbaa [[INT_TBAA2]] +// CHECK-NEXT: ret void +// +// AIX-LABEL: define void @test_unsigned_lwat_cond( +// AIX-SAME: ptr noundef [[PTR:%.*]], ptr noundef writeonly captures(none) initializes((0, 4)) [[RESP:%.*]]) local_unnamed_addr #[[ATTR0]] { +// AIX-NEXT: [[ENTRY:.*:]] +// AIX-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.ppc.amo.lwat.cond(ptr [[PTR]], i32 24) +// AIX-NEXT: store i32 [[TMP0]], ptr [[RESP]], align 4, !tbaa [[INT_TBAA2]] +// AIX-NEXT: ret void +// +void test_unsigned_lwat_cond(unsigned int *ptr, unsigned int * resp) { + unsigned int res = __builtin_amo_lwat_cond(ptr, 24); + *resp = res; +} + +// CHECK-LABEL: define dso_local void @test_unsigned_ldat_cond( +// CHECK-SAME: ptr noundef [[PTR:%.*]], ptr noundef writeonly captures(none) initializes((0, 8)) [[RESP:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.ppc.amo.ldat.cond(ptr [[PTR]], i32 25) +// CHECK-NEXT: store i64 [[TMP0]], ptr [[RESP]], align 8, !tbaa [[LONG_TBAA6]] +// CHECK-NEXT: ret void +// +// AIX-LABEL: define void @test_unsigned_ldat_cond( +// AIX-SAME: ptr noundef [[PTR:%.*]], ptr noundef writeonly captures(none) initializes((0, 8)) [[RESP:%.*]]) local_unnamed_addr #[[ATTR0]] { +// AIX-NEXT: [[ENTRY:.*:]] +// AIX-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.ppc.amo.ldat.cond(ptr [[PTR]], i32 25) +// AIX-NEXT: store i64 [[TMP0]], ptr [[RESP]], align 8, !tbaa [[LONG_TBAA6]] +// AIX-NEXT: ret void +// +void test_unsigned_ldat_cond(unsigned long int *ptr, unsigned long int * resp) { + unsigned long int res = __builtin_amo_ldat_cond(ptr, 25); + *resp = res; +} + +// CHECK-LABEL: define dso_local void @test_signed_lwat_cond( +// CHECK-SAME: ptr noundef [[PTR:%.*]], ptr noundef writeonly captures(none) initializes((0, 4)) [[RESP:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.ppc.amo.lwat.cond(ptr [[PTR]], i32 24) +// CHECK-NEXT: store i32 [[TMP0]], ptr [[RESP]], align 4, !tbaa [[INT_TBAA2]] +// CHECK-NEXT: ret void +// +// AIX-LABEL: define void @test_signed_lwat_cond( +// AIX-SAME: ptr noundef [[PTR:%.*]], ptr noundef writeonly captures(none) initializes((0, 4)) [[RESP:%.*]]) local_unnamed_addr #[[ATTR0]] { +// AIX-NEXT: [[ENTRY:.*:]] +// AIX-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.ppc.amo.lwat.cond(ptr [[PTR]], i32 24) +// AIX-NEXT: store i32 [[TMP0]], ptr [[RESP]], align 4, !tbaa [[INT_TBAA2]] +// AIX-NEXT: ret void +// +void test_signed_lwat_cond(int *ptr, int * resp) { + int res = __builtin_amo_lwat_cond_s(ptr, 24); + *resp = res; +} + +// CHECK-LABEL: define dso_local void @test_signed_ldat_cond( +// CHECK-SAME: ptr noundef [[PTR:%.*]], ptr noundef writeonly captures(none) initializes((0, 8)) [[RESP:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-NEXT: [[ENTRY:.*:]] +// CHECK-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.ppc.amo.ldat.cond(ptr [[PTR]], i32 25) +// CHECK-NEXT: store i64 [[TMP0]], ptr [[RESP]], align 8, !tbaa [[LONG_TBAA6]] +// CHECK-NEXT: ret void +// +// AIX-LABEL: define void @test_signed_ldat_cond( +// AIX-SAME: ptr noundef [[PTR:%.*]], ptr noundef writeonly captures(none) initializes((0, 8)) [[RESP:%.*]]) local_unnamed_addr #[[ATTR0]] { +// AIX-NEXT: [[ENTRY:.*:]] +// AIX-NEXT: [[TMP0:%.*]] = tail call i64 @llvm.ppc.amo.ldat.cond(ptr [[PTR]], i32 25) +// AIX-NEXT: store i64 [[TMP0]], ptr [[RESP]], align 8, !tbaa [[LONG_TBAA6]] +// AIX-NEXT: ret void +// +void test_signed_ldat_cond(long int *ptr, long int * resp) { + long int res = __builtin_amo_ldat_cond_s(ptr, 25); + *resp = res; +} + //. // CHECK: [[INT_TBAA2]] = !{[[META3:![0-9]+]], [[META3]], i64 0} // CHECK: [[META3]] = !{!"int", [[META4:![0-9]+]], i64 0} diff --git a/llvm/include/llvm/IR/IntrinsicsPowerPC.td b/llvm/include/llvm/IR/IntrinsicsPowerPC.td index 446113c4670dd..e13f40454fba2 100644 --- a/llvm/include/llvm/IR/IntrinsicsPowerPC.td +++ b/llvm/include/llvm/IR/IntrinsicsPowerPC.td @@ -2150,4 +2150,12 @@ let TargetPrefix = "ppc" in { DefaultAttrsIntrinsic<[llvm_i64_ty],[llvm_ptr_ty, llvm_i64_ty, llvm_i32_ty], [IntrArgMemOnly, ImmArg>]>; + def int_ppc_amo_lwat_cond : ClangBuiltin<"__builtin_amo_lwat_cond">, + DefaultAttrsIntrinsic<[llvm_i32_ty],[llvm_ptr_ty, + llvm_i32_ty], + [IntrArgMemOnly, ImmArg>]>; + def int_ppc_amo_ldat_cond : ClangBuiltin<"__builtin_amo_ldat_cond">, + DefaultAttrsIntrinsic<[llvm_i64_ty],[llvm_ptr_ty, + llvm_i32_ty], + [IntrArgMemOnly, ImmArg>]>; } diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp index ec5e61b724d87..666495ef920b0 100644 --- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp +++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp @@ -11641,7 +11641,7 @@ SDValue PPCTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op, SDLoc dl(Op); switch (IntrinsicID) { case Intrinsic::ppc_amo_lwat: - case Intrinsic::ppc_amo_ldat: + case Intrinsic::ppc_amo_ldat: { SDValue Ptr = Op.getOperand(2); SDValue Val1 = Op.getOperand(3); SDValue FC = Op.getOperand(4); @@ -11654,6 +11654,21 @@ SDValue PPCTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op, SDValue OutChain = SDValue(MNode, 1); return DAG.getMergeValues({Result, OutChain}, dl); } + case Intrinsic::ppc_amo_lwat_cond: + case Intrinsic::ppc_amo_ldat_cond: { + SDValue Ptr = Op.getOperand(2); + SDValue FC = Op.getOperand(3); + SDValue Ops[] = {Ptr, FC}; + bool IsLwat_cond = IntrinsicID == Intrinsic::ppc_amo_lwat_cond; + unsigned Opcode = + IsLwat_cond ? PPC::LWAT_COND_PSEUDO : PPC::LDAT_COND_PSEUDO; + MachineSDNode *MNode = DAG.getMachineNode( + Opcode, dl, {IsLwat_cond ? MVT::i32 : MVT::i64, MVT::Other}, Ops); + SDValue Result = SDValue(MNode, 0); + SDValue OutChain = SDValue(MNode, 1); + return DAG.getMergeValues({Result, OutChain}, dl); + } + } return SDValue(); } @@ -14777,6 +14792,32 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, else BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY), DstReg) .addReg(Result64); + } else if (MI.getOpcode() == PPC::LWAT_COND_PSEUDO || + MI.getOpcode() == PPC::LDAT_COND_PSEUDO) { + DebugLoc DL = MI.getDebugLoc(); + Register DstReg = MI.getOperand(0).getReg(); + Register PtrReg = MI.getOperand(1).getReg(); + unsigned FC = MI.getOperand(2).getImm(); + bool IsLwat_Cond = MI.getOpcode() == PPC::LWAT_COND_PSEUDO; + + Register Pair = MRI.createVirtualRegister(&PPC::G8pRCRegClass); + BuildMI(*BB, MI, DL, TII->get(TargetOpcode::IMPLICIT_DEF), Pair); + + Register PairResult = MRI.createVirtualRegister(&PPC::G8pRCRegClass); + BuildMI(*BB, MI, DL, TII->get(IsLwat_Cond ? PPC::LWAT : PPC::LDAT), + PairResult) + .addReg(Pair) + .addReg(PtrReg) + .addImm(FC); + Register Result64 = MRI.createVirtualRegister(&PPC::G8RCRegClass); + BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY), Result64) + .addReg(PairResult, 0, PPC::sub_gp8_x0); + if (IsLwat_Cond) + BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY), DstReg) + .addReg(Result64, 0, PPC::sub_32); + else + BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY), DstReg) + .addReg(Result64); } else { llvm_unreachable("Unexpected instr type to insert"); } diff --git a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td index 462535601e05e..6e820a12b97eb 100644 --- a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td +++ b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td @@ -340,6 +340,12 @@ def LDAT_PSEUDO : PPCCustomInserterPseudo< "#LDAT_PSEUDO", []>; +def LDAT_COND_PSEUDO : PPCCustomInserterPseudo < + (outs g8rc:$dst), + (ins ptr_rc_nor0:$ptr, u5imm:$fc), + "#LDAT_COND_PSEUDO", + []>; + let Defs = [CR0], mayStore = 1, mayLoad = 0, hasSideEffects = 0 in { def STDCX : XForm_1_memOp<31, 214, (outs), (ins g8rc:$RST, (memrr $RA, $RB):$addr), "stdcx. $RST, $addr", IIC_LdStSTDCX, []>, isRecordForm; diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td index 3fddda2535673..29ca44287ef10 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td +++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td @@ -1879,6 +1879,12 @@ def LWAT_PSEUDO : PPCCustomInserterPseudo< "#LWAT_PSEUDO", []>; +def LWAT_COND_PSEUDO : PPCCustomInserterPseudo < + (outs gprc:$dst), + (ins ptr_rc_nor0:$ptr, u5imm:$fc), + "#LWAT_COND_PSEUDO", + []>; + let Defs = [CR0], mayStore = 1, mayLoad = 0, hasSideEffects = 0 in { def STBCX : XForm_1_memOp<31, 694, (outs), (ins gprc:$RST, (memrr $RA, $RB):$addr), "stbcx. $RST, $addr", IIC_LdStSTWCX, []>, diff --git a/llvm/test/CodeGen/PowerPC/amo-enable.ll b/llvm/test/CodeGen/PowerPC/amo-enable.ll index 37c91e2ff5505..0c24057c0d342 100644 --- a/llvm/test/CodeGen/PowerPC/amo-enable.ll +++ b/llvm/test/CodeGen/PowerPC/amo-enable.ll @@ -86,5 +86,43 @@ entry: ret void } +define void @test_lwat_cond(ptr noundef %ptr, ptr nocapture %resp) { +; CHECK-LABEL: test_lwat_cond: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: lwat r6, r3, 24 +; CHECK-NEXT: stw r6, 0(r4) +; CHECK-NEXT: blr +; +; CHECK-BE-LABEL: test_lwat_cond: +; CHECK-BE: # %bb.0: # %entry +; CHECK-BE-NEXT: lwat r6, r3, 24 +; CHECK-BE-NEXT: stw r6, 0(r4) +; CHECK-BE-NEXT: blr +entry: + %0 = tail call i32 @llvm.ppc.amo.lwat.cond(ptr %ptr, i32 24) + store i32 %0, ptr %resp, align 4 + ret void +} + +define void @test_ldat_cond(ptr noundef %ptr, ptr nocapture %resp) { +; CHECK-LABEL: test_ldat_cond: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: ldat r6, r3, 24 +; CHECK-NEXT: std r6, 0(r4) +; CHECK-NEXT: blr +; +; CHECK-BE-LABEL: test_ldat_cond: +; CHECK-BE: # %bb.0: # %entry +; CHECK-BE-NEXT: ldat r6, r3, 24 +; CHECK-BE-NEXT: std r6, 0(r4) +; CHECK-BE-NEXT: blr +entry: + %0 = tail call i64 @llvm.ppc.amo.ldat.cond(ptr %ptr, i32 24) + store i64 %0, ptr %resp, align 8 + ret void +} + declare i64 @llvm.ppc.amo.ldat(ptr, i64, i32 immarg) declare i32 @llvm.ppc.amo.lwat(ptr, i32, i32 immarg) +declare i64 @llvm.ppc.amo.ldat.cond(ptr, i32 immarg) +declare i32 @llvm.ppc.amo.lwat.cond(ptr, i32 immarg)