diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp index 9aa1c31db7d20..06a37f1f559d4 100644 --- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp @@ -190,8 +190,14 @@ std::string lookupBuiltinNameHelper(StringRef DemangledCall) { // Check if the extracted name contains type information between angle // brackets. If so, the builtin is an instantiated template - needs to have // the information after angle brackets and return type removed. - if (BuiltinName.find('<') && BuiltinName.back() == '>') { - BuiltinName = BuiltinName.substr(0, BuiltinName.find('<')); + std::size_t Pos1 = BuiltinName.rfind('<'); + if (Pos1 != std::string::npos && BuiltinName.back() == '>') { + std::size_t Pos2 = BuiltinName.rfind(' ', Pos1); + if (Pos2 == std::string::npos) + Pos2 = 0; + else + ++Pos2; + BuiltinName = BuiltinName.substr(Pos2, Pos1 - Pos2); BuiltinName = BuiltinName.substr(BuiltinName.find_last_of(' ') + 1); } @@ -461,9 +467,11 @@ static Register buildBuiltinVariableLoad( SPIRVGlobalRegistry *GR, SPIRV::BuiltIn::BuiltIn BuiltinValue, LLT LLType, Register Reg = Register(0), bool isConst = true, bool hasLinkageTy = true) { Register NewRegister = - MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::iIDRegClass); - MIRBuilder.getMRI()->setType(NewRegister, - LLT::pointer(0, GR->getPointerSize())); + MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::pIDRegClass); + MIRBuilder.getMRI()->setType( + NewRegister, + LLT::pointer(storageClassToAddressSpace(SPIRV::StorageClass::Function), + GR->getPointerSize())); SPIRVType *PtrType = GR->getOrCreateSPIRVPointerType( VariableType, MIRBuilder, SPIRV::StorageClass::Input); GR->assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF()); @@ -1556,6 +1564,55 @@ static bool generateWaveInst(const SPIRV::IncomingCall *Call, /* isConst= */ false, /* hasLinkageTy= */ false); } +// We expect a builtin +// Name(ptr sret([RetType]) %result, Type %operand1, Type %operand1) +// where %result is a pointer to where the result of the builtin execution +// is to be stored, and generate the following instructions: +// Res = Opcode RetType Operand1 Operand1 +// OpStore RetVariable Res +static bool generateICarryBorrowInst(const SPIRV::IncomingCall *Call, + MachineIRBuilder &MIRBuilder, + SPIRVGlobalRegistry *GR) { + const SPIRV::DemangledBuiltin *Builtin = Call->Builtin; + unsigned Opcode = + SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode; + + Register SRetReg = Call->Arguments[0]; + SPIRVType *PtrRetType = GR->getSPIRVTypeForVReg(SRetReg); + SPIRVType *RetType = GR->getPointeeType(PtrRetType); + if (!RetType) + report_fatal_error("The first parameter must be a pointer"); + if (RetType->getOpcode() != SPIRV::OpTypeStruct) + report_fatal_error("Expected struct type result for the arithmetic with " + "overflow builtins"); + + SPIRVType *OpType1 = GR->getSPIRVTypeForVReg(Call->Arguments[1]); + SPIRVType *OpType2 = GR->getSPIRVTypeForVReg(Call->Arguments[2]); + if (!OpType1 || !OpType2 || OpType1 != OpType2) + report_fatal_error("Operands must have the same type"); + if (OpType1->getOpcode() == SPIRV::OpTypeVector) + switch (Opcode) { + case SPIRV::OpIAddCarryS: + Opcode = SPIRV::OpIAddCarryV; + break; + case SPIRV::OpISubBorrowS: + Opcode = SPIRV::OpISubBorrowV; + break; + } + + MachineRegisterInfo *MRI = MIRBuilder.getMRI(); + Register ResReg = MRI->createGenericVirtualRegister(LLT::scalar(64)); + MRI->setRegClass(ResReg, &SPIRV::iIDRegClass); + GR->assignSPIRVTypeToVReg(RetType, ResReg, MIRBuilder.getMF()); + MIRBuilder.buildInstr(Opcode) + .addDef(ResReg) + .addUse(GR->getSPIRVTypeID(RetType)) + .addUse(Call->Arguments[1]) + .addUse(Call->Arguments[2]); + MIRBuilder.buildInstr(SPIRV::OpStore).addUse(SRetReg).addUse(ResReg); + return true; +} + static bool generateGetQueryInst(const SPIRV::IncomingCall *Call, MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR) { @@ -2511,6 +2568,8 @@ std::optional lowerBuiltin(const StringRef DemangledCall, return generateDotOrFMulInst(Call.get(), MIRBuilder, GR); case SPIRV::Wave: return generateWaveInst(Call.get(), MIRBuilder, GR); + case SPIRV::ICarryBorrow: + return generateICarryBorrowInst(Call.get(), MIRBuilder, GR); case SPIRV::GetQuery: return generateGetQueryInst(Call.get(), MIRBuilder, GR); case SPIRV::ImageSizeQuery: diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td index 29f11c3dd3686..1b95b1479bb93 100644 --- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td +++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td @@ -63,6 +63,7 @@ def KernelClock : BuiltinGroup; def CastToPtr : BuiltinGroup; def Construct : BuiltinGroup; def CoopMatr : BuiltinGroup; +def ICarryBorrow : BuiltinGroup; //===----------------------------------------------------------------------===// // Class defining a demangled builtin record. The information in the record @@ -628,6 +629,10 @@ defm : DemangledNativeBuiltin<"barrier", OpenCL_std, Barrier, 1, 3, OpControlBar defm : DemangledNativeBuiltin<"work_group_barrier", OpenCL_std, Barrier, 1, 3, OpControlBarrier>; defm : DemangledNativeBuiltin<"__spirv_ControlBarrier", OpenCL_std, Barrier, 3, 3, OpControlBarrier>; +// ICarryBorrow builtin record: +defm : DemangledNativeBuiltin<"__spirv_IAddCarry", OpenCL_std, ICarryBorrow, 3, 3, OpIAddCarryS>; +defm : DemangledNativeBuiltin<"__spirv_ISubBorrow", OpenCL_std, ICarryBorrow, 3, 3, OpISubBorrowS>; + // cl_intel_split_work_group_barrier defm : DemangledNativeBuiltin<"intel_work_group_barrier_arrive", OpenCL_std, Barrier, 1, 2, OpControlBarrierArriveINTEL>; defm : DemangledNativeBuiltin<"__spirv_ControlBarrierArriveINTEL", OpenCL_std, Barrier, 3, 3, OpControlBarrierArriveINTEL>; diff --git a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp index 98cf598a1f031..3c5397319aaf2 100644 --- a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp @@ -546,12 +546,36 @@ bool SPIRVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder, ArgVRegs.push_back(ArgReg); SPIRVType *SpvType = GR->getSPIRVTypeForVReg(ArgReg); if (!SpvType) { - SpvType = GR->getOrCreateSPIRVType(Arg.Ty, MIRBuilder); - GR->assignSPIRVTypeToVReg(SpvType, ArgReg, MF); + Type *ArgTy = nullptr; + if (auto *PtrArgTy = dyn_cast(Arg.Ty)) { + // If Arg.Ty is an untyped pointer (i.e., ptr [addrspace(...)]) and we + // don't have access to original value in LLVM IR or info about + // deduced pointee type, then we should wait with setting the type for + // the virtual register until pre-legalizer step when we access + // @llvm.spv.assign.ptr.type.p...(...)'s info. + if (Arg.OrigValue) + if (Type *ElemTy = GR->findDeducedElementType(Arg.OrigValue)) + ArgTy = + TypedPointerType::get(ElemTy, PtrArgTy->getAddressSpace()); + } else { + ArgTy = Arg.Ty; + } + if (ArgTy) { + SpvType = GR->getOrCreateSPIRVType(ArgTy, MIRBuilder); + GR->assignSPIRVTypeToVReg(SpvType, ArgReg, MF); + } } if (!MRI->getRegClassOrNull(ArgReg)) { - MRI->setRegClass(ArgReg, GR->getRegClass(SpvType)); - MRI->setType(ArgReg, GR->getRegType(SpvType)); + // Either we have SpvType created, or Arg.Ty is an untyped pointer and + // we know its virtual register's class and type even if we don't know + // pointee type. + MRI->setRegClass(ArgReg, SpvType ? GR->getRegClass(SpvType) + : &SPIRV::pIDRegClass); + MRI->setType( + ArgReg, + SpvType ? GR->getRegType(SpvType) + : LLT::pointer(cast(Arg.Ty)->getAddressSpace(), + GR->getPointerSize())); } } auto instructionSet = canUseOpenCL ? SPIRV::InstructionSet::OpenCL_std diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp index 8b7e9c48de6c7..e6ef40e010dc2 100644 --- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp @@ -76,8 +76,8 @@ class SPIRVEmitIntrinsics SPIRV::InstructionSet::InstructionSet InstrSet; // a register of Instructions that don't have a complete type definition - SmallPtrSet UncompleteTypeInfo; - SmallVector PostprocessWorklist; + DenseMap UncompleteTypeInfo; + SmallVector PostprocessWorklist; // well known result types of builtins enum WellKnownTypes { Event }; @@ -147,6 +147,7 @@ class SPIRVEmitIntrinsics std::unordered_set &FVisited); void replaceWithPtrcasted(Instruction *CI, Type *NewElemTy, Type *KnownElemTy, CallInst *AssignCI); + void replaceAllUsesWith(Value *Src, Value *Dest, bool DeleteOld = true); bool runOnFunction(Function &F); bool postprocessTypes(); @@ -272,6 +273,27 @@ static inline void reportFatalOnTokenType(const Instruction *I) { false); } +void SPIRVEmitIntrinsics::replaceAllUsesWith(Value *Src, Value *Dest, + bool DeleteOld) { + Src->replaceAllUsesWith(Dest); + // Update deduced type records + GR->updateIfExistDeducedElementType(Src, Dest, DeleteOld); + GR->updateIfExistAssignPtrTypeInstr(Src, Dest, DeleteOld); + // Update uncomplete type records if any + auto It = UncompleteTypeInfo.find(Src); + if (It == UncompleteTypeInfo.end()) + return; + if (DeleteOld) { + unsigned Pos = It->second; + UncompleteTypeInfo.erase(Src); + UncompleteTypeInfo[Dest] = Pos; + PostprocessWorklist[Pos] = Dest; + } else { + UncompleteTypeInfo[Dest] = PostprocessWorklist.size(); + PostprocessWorklist.push_back(Dest); + } +} + static bool IsKernelArgInt8(Function *F, StoreInst *SI) { return SI && F->getCallingConv() == CallingConv::SPIR_KERNEL && isPointerTy(SI->getValueOperand()->getType()) && @@ -434,7 +456,7 @@ void SPIRVEmitIntrinsics::maybeAssignPtrType(Type *&Ty, Value *Op, Type *RefTy, if (!UnknownElemTypeI8) return; if (auto *I = dyn_cast(Op)) { - UncompleteTypeInfo.insert(I); + UncompleteTypeInfo[I] = PostprocessWorklist.size(); PostprocessWorklist.push_back(I); } } @@ -640,7 +662,7 @@ Type *SPIRVEmitIntrinsics::deduceElementType(Value *I, bool UnknownElemTypeI8) { if (!UnknownElemTypeI8) return nullptr; if (auto *Instr = dyn_cast(I)) { - UncompleteTypeInfo.insert(Instr); + UncompleteTypeInfo[Instr] = PostprocessWorklist.size(); PostprocessWorklist.push_back(Instr); } return IntegerType::getInt8Ty(I->getContext()); @@ -1062,7 +1084,7 @@ Instruction *SPIRVEmitIntrinsics::visitSwitchInst(SwitchInst &I) { {I.getOperand(0)->getType()}, {Args}); // remove switch to avoid its unneeded and undesirable unwrap into branches // and conditions - I.replaceAllUsesWith(NewI); + replaceAllUsesWith(&I, NewI); I.eraseFromParent(); // insert artificial and temporary instruction to preserve valid CFG, // it will be removed after IR translation pass @@ -1084,7 +1106,7 @@ Instruction *SPIRVEmitIntrinsics::visitGetElementPtrInst(GetElementPtrInst &I) { for (auto &Op : I.operands()) Args.push_back(Op); auto *NewI = B.CreateIntrinsic(Intrinsic::spv_gep, {Types}, {Args}); - I.replaceAllUsesWith(NewI); + replaceAllUsesWith(&I, NewI); I.eraseFromParent(); return NewI; } @@ -1099,7 +1121,7 @@ Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) { // such bitcasts do not provide sufficient information, should be just skipped // here, and handled in insertPtrCastOrAssignTypeInstr. if (isPointerTy(I.getType())) { - I.replaceAllUsesWith(Source); + replaceAllUsesWith(&I, Source); I.eraseFromParent(); return nullptr; } @@ -1108,7 +1130,7 @@ Instruction *SPIRVEmitIntrinsics::visitBitCastInst(BitCastInst &I) { SmallVector Args(I.op_begin(), I.op_end()); auto *NewI = B.CreateIntrinsic(Intrinsic::spv_bitcast, {Types}, {Args}); std::string InstName = I.hasName() ? I.getName().str() : ""; - I.replaceAllUsesWith(NewI); + replaceAllUsesWith(&I, NewI); I.eraseFromParent(); NewI->setName(InstName); return NewI; @@ -1219,6 +1241,8 @@ void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast( SmallVector Args = {Pointer, VMD, B.getInt32(AddressSpace)}; auto *PtrCastI = B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args); I->setOperand(OperandToReplace, PtrCastI); + // We need to set up a pointee type for the newly created spv_ptrcast. + buildAssignPtr(B, ExpectedElementType, PtrCastI); } void SPIRVEmitIntrinsics::insertPtrCastOrAssignTypeInstr(Instruction *I, @@ -1331,7 +1355,7 @@ Instruction *SPIRVEmitIntrinsics::visitInsertElementInst(InsertElementInst &I) { SmallVector Args(I.op_begin(), I.op_end()); auto *NewI = B.CreateIntrinsic(Intrinsic::spv_insertelt, {Types}, {Args}); std::string InstName = I.hasName() ? I.getName().str() : ""; - I.replaceAllUsesWith(NewI); + replaceAllUsesWith(&I, NewI); I.eraseFromParent(); NewI->setName(InstName); return NewI; @@ -1346,7 +1370,7 @@ SPIRVEmitIntrinsics::visitExtractElementInst(ExtractElementInst &I) { SmallVector Args = {I.getVectorOperand(), I.getIndexOperand()}; auto *NewI = B.CreateIntrinsic(Intrinsic::spv_extractelt, {Types}, {Args}); std::string InstName = I.hasName() ? I.getName().str() : ""; - I.replaceAllUsesWith(NewI); + replaceAllUsesWith(&I, NewI); I.eraseFromParent(); NewI->setName(InstName); return NewI; @@ -1382,7 +1406,7 @@ Instruction *SPIRVEmitIntrinsics::visitExtractValueInst(ExtractValueInst &I) { Args.push_back(B.getInt32(Op)); auto *NewI = B.CreateIntrinsic(Intrinsic::spv_extractv, {I.getType()}, {Args}); - I.replaceAllUsesWith(NewI); + replaceAllUsesWith(&I, NewI); I.eraseFromParent(); return NewI; } @@ -1443,7 +1467,7 @@ Instruction *SPIRVEmitIntrinsics::visitAllocaInst(AllocaInst &I) { {PtrTy, ArraySize->getType()}, {ArraySize}) : B.CreateIntrinsic(Intrinsic::spv_alloca, {PtrTy}, {}); std::string InstName = I.hasName() ? I.getName().str() : ""; - I.replaceAllUsesWith(NewI); + replaceAllUsesWith(&I, NewI); I.eraseFromParent(); NewI->setName(InstName); return NewI; @@ -1613,7 +1637,7 @@ void SPIRVEmitIntrinsics::processInstrAfterVisit(Instruction *I, auto *NewOp = buildIntrWithMD(Intrinsic::spv_track_constant, {II->getType(), II->getType()}, t->second, I, {}, B); - I->replaceAllUsesWith(NewOp); + replaceAllUsesWith(I, NewOp, false); NewOp->setArgOperand(0, I); } bool IsPhi = isa(I), BPrepared = false; diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h index a95b488960c4c..2805300ae4437 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h @@ -180,6 +180,15 @@ class SPIRVGlobalRegistry { auto It = AssignPtrTypeInstr.find(Val); return It == AssignPtrTypeInstr.end() ? nullptr : It->second; } + // - Find a record and update its key or add a new record, if found. + void updateIfExistAssignPtrTypeInstr(Value *OldVal, Value *NewVal, + bool DeleteOld) { + if (CallInst *CI = findAssignPtrTypeInstr(OldVal)) { + if (DeleteOld) + AssignPtrTypeInstr.erase(OldVal); + AssignPtrTypeInstr[NewVal] = CI; + } + } // A registry of mutated values // (see `SPIRVPrepareFunctions::removeAggregateTypesFromSignature()`): @@ -214,6 +223,15 @@ class SPIRVGlobalRegistry { auto It = DeducedElTys.find(Val); return It == DeducedElTys.end() ? nullptr : It->second; } + // - Find a record and update its key or add a new record, if found. + void updateIfExistDeducedElementType(Value *OldVal, Value *NewVal, + bool DeleteOld) { + if (Type *Ty = findDeducedElementType(OldVal)) { + if (DeleteOld) + DeducedElTys.erase(OldVal); + DeducedElTys[NewVal] = Ty; + } + } // - Add a record to the map of deduced composite types. void addDeducedCompositeType(Value *Val, Type *Ty) { DeducedNestedTys[Val] = Ty; diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp index ecbceb5b472fa..59a1bf50b771b 100644 --- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp @@ -414,6 +414,17 @@ void SPIRVTargetLowering::finalizeLowering(MachineFunction &MF) const { validateForwardCalls(STI, MRI, GR, MI); break; + // ensure that LLVM IR add/sub instructions result in logical SPIR-V + // instructions when applied to bool type + case SPIRV::OpIAddS: + case SPIRV::OpIAddV: + case SPIRV::OpISubS: + case SPIRV::OpISubV: + if (GR.isScalarOrVectorOfType(MI.getOperand(1).getReg(), + SPIRV::OpTypeBool)) + MI.setDesc(STI.getInstrInfo()->get(SPIRV::OpLogicalNotEqual)); + break; + // ensure that LLVM IR bitwise instructions result in logical SPIR-V // instructions when applied to bool type case SPIRV::OpBitwiseOrS: @@ -473,8 +484,11 @@ void SPIRVTargetLowering::finalizeLowering(MachineFunction &MF) const { MI.getOperand(2).getImm() != SPIRV::InstructionSet::OpenCL_std) continue; switch (MI.getOperand(3).getImm()) { + case SPIRV::OpenCLExtInst::frexp: + case SPIRV::OpenCLExtInst::lgamma_r: case SPIRV::OpenCLExtInst::remquo: { - // The last operand must be of a pointer to the return type. + // The last operand must be of a pointer to i32 or vector of i32 + // values. MachineIRBuilder MIB(MI); SPIRVType *Int32Type = GR.getOrCreateSPIRVIntegerType(32, MIB); SPIRVType *RetType = MRI->getVRegDef(MI.getOperand(1).getReg()); @@ -487,8 +501,6 @@ void SPIRVTargetLowering::finalizeLowering(MachineFunction &MF) const { Int32Type, RetType->getOperand(2).getImm(), MIB)); } break; case SPIRV::OpenCLExtInst::fract: - case SPIRV::OpenCLExtInst::frexp: - case SPIRV::OpenCLExtInst::lgamma_r: case SPIRV::OpenCLExtInst::modf: case SPIRV::OpenCLExtInst::sincos: // The last operand must be of a pointer to the base type represented diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index a968b65c17306..5c13450893a04 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -1170,7 +1170,7 @@ bool SPIRVInstructionSelector::selectOverflowArith(Register ResVReg, // "Result Type must be from OpTypeStruct. The struct must have two members, // and the two members must be the same type." Type *ResElemTy = cast(ResTy)->getElementType(0); - ResTy = StructType::create(SmallVector{ResElemTy, ResElemTy}); + ResTy = StructType::get(ResElemTy, ResElemTy); // Build SPIR-V types and constant(s) if needed. MachineIRBuilder MIRBuilder(I); SPIRVType *StructType = GR.getOrCreateSPIRVType( @@ -3051,22 +3051,9 @@ bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg, MachineInstr &I) const { // Change order of instructions if needed: all OpVariable instructions in a // function must be the first instructions in the first block - MachineFunction *MF = I.getParent()->getParent(); - MachineBasicBlock *MBB = &MF->front(); - auto It = MBB->SkipPHIsAndLabels(MBB->begin()), E = MBB->end(); - bool IsHeader = false; - unsigned Opcode; - for (; It != E && It != I; ++It) { - Opcode = It->getOpcode(); - if (Opcode == SPIRV::OpFunction || Opcode == SPIRV::OpFunctionParameter) { - IsHeader = true; - } else if (IsHeader && - !(Opcode == SPIRV::ASSIGN_TYPE || Opcode == SPIRV::OpLabel)) { - ++It; - break; - } - } - return BuildMI(*MBB, It, It->getDebugLoc(), TII.get(SPIRV::OpVariable)) + auto It = getOpVariableMBBIt(I); + return BuildMI(*It->getParent(), It, It->getDebugLoc(), + TII.get(SPIRV::OpVariable)) .addDef(ResVReg) .addUse(GR.getSPIRVTypeID(ResType)) .addImm(static_cast(SPIRV::StorageClass::Function)) diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp index f9b361e163c90..aeb2c29f7b861 100644 --- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp @@ -162,6 +162,26 @@ void buildOpSpirvDecorations(Register Reg, MachineIRBuilder &MIRBuilder, } } +MachineBasicBlock::iterator getOpVariableMBBIt(MachineInstr &I) { + MachineFunction *MF = I.getParent()->getParent(); + MachineBasicBlock *MBB = &MF->front(); + MachineBasicBlock::iterator It = MBB->SkipPHIsAndLabels(MBB->begin()), + E = MBB->end(); + bool IsHeader = false; + unsigned Opcode; + for (; It != E && It != I; ++It) { + Opcode = It->getOpcode(); + if (Opcode == SPIRV::OpFunction || Opcode == SPIRV::OpFunctionParameter) { + IsHeader = true; + } else if (IsHeader && + !(Opcode == SPIRV::ASSIGN_TYPE || Opcode == SPIRV::OpLabel)) { + ++It; + break; + } + } + return It; +} + SPIRV::StorageClass::StorageClass addressSpaceToStorageClass(unsigned AddrSpace, const SPIRVSubtarget &STI) { switch (AddrSpace) { diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.h b/llvm/lib/Target/SPIRV/SPIRVUtils.h index 11fd3a5c61dca..298b0b93b0e4d 100644 --- a/llvm/lib/Target/SPIRV/SPIRVUtils.h +++ b/llvm/lib/Target/SPIRV/SPIRVUtils.h @@ -15,6 +15,7 @@ #include "MCTargetDesc/SPIRVBaseInfo.h" #include "llvm/Analysis/LoopInfo.h" +#include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/TypedPointerType.h" @@ -139,6 +140,10 @@ void buildOpDecorate(Register Reg, MachineInstr &I, const SPIRVInstrInfo &TII, void buildOpSpirvDecorations(Register Reg, MachineIRBuilder &MIRBuilder, const MDNode *GVarMD); +// Return a valid position for the OpVariable instruction inside a function, +// i.e., at the beginning of the first block of the function. +MachineBasicBlock::iterator getOpVariableMBBIt(MachineInstr &I); + // Convert a SPIR-V storage class to the corresponding LLVM IR address space. // TODO: maybe the following two functions should be handled in the subtarget // to allow for different OpenCL vs Vulkan handling. diff --git a/llvm/test/CodeGen/SPIRV/iaddcarry-builtin.ll b/llvm/test/CodeGen/SPIRV/iaddcarry-builtin.ll new file mode 100644 index 0000000000000..8f14eba21b63a --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/iaddcarry-builtin.ll @@ -0,0 +1,129 @@ +; Adapted from Khronos Translator test suite: test/iaddcarry_builtin.ll + +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +%i8struct = type {i8, i8} +%i16struct = type {i16, i16} +%i32struct = type {i32, i32} +%i64struct = type {i64, i64} +%vecstruct = type {<4 x i32>, <4 x i32>} + +; CHECK-SPIRV-DAG: [[uchar:%[a-z0-9_]+]] = OpTypeInt 8 +; CHECK-SPIRV-DAG: [[ushort:%[a-z0-9_]+]] = OpTypeInt 16 +; CHECK-SPIRV-DAG: [[uint:%[a-z0-9_]+]] = OpTypeInt 32 +; CHECK-SPIRV-DAG: [[ulong:%[a-z0-9_]+]] = OpTypeInt 64 +; CHECK-SPIRV-DAG: [[void:%[a-z0-9_]+]] = OpTypeVoid +; CHECK-SPIRV-DAG: [[i8struct:%[a-z0-9_]+]] = OpTypeStruct [[uchar]] [[uchar]] +; CHECK-SPIRV-DAG: [[_ptr_Function_i8struct:%[a-z0-9_]+]] = OpTypePointer Function [[i8struct]] +; CHECK-SPIRV-DAG: [[i16struct:%[a-z0-9_]+]] = OpTypeStruct [[ushort]] [[ushort]] +; CHECK-SPIRV-DAG: [[_ptr_Function_i16struct:%[a-z0-9_]+]] = OpTypePointer Function [[i16struct]] +; CHECK-SPIRV-DAG: [[i32struct:%[a-z0-9_]+]] = OpTypeStruct [[uint]] [[uint]] +; CHECK-SPIRV-DAG: [[_ptr_Function_i32struct:%[a-z0-9_]+]] = OpTypePointer Function [[i32struct]] +; CHECK-SPIRV-DAG: [[i64struct:%[a-z0-9_]+]] = OpTypeStruct [[ulong]] [[ulong]] +; CHECK-SPIRV-DAG: [[_ptr_Function_i64struct:%[a-z0-9_]+]] = OpTypePointer Function [[i64struct]] +; CHECK-SPIRV-DAG: [[v4uint:%[a-z0-9_]+]] = OpTypeVector [[uint]] 4 +; CHECK-SPIRV-DAG: [[vecstruct:%[a-z0-9_]+]] = OpTypeStruct [[v4uint]] [[v4uint]] +; CHECK-SPIRV-DAG: [[_ptr_Function_vecstruct:%[a-z0-9_]+]] = OpTypePointer Function [[vecstruct]] +; CHECK-SPIRV-DAG: [[struct_anon:%[a-z0-9_.]+]] = OpTypeStruct [[uint]] [[uint]] +; CHECK-SPIRV-DAG: [[_ptr_Function_struct_anon:%[a-z0-9_]+]] = OpTypePointer Function [[struct_anon]] +; CHECK-SPIRV-DAG: [[_ptr_Generic_struct_anon:%[a-z0-9_]+]] = OpTypePointer Generic [[struct_anon]] + +define spir_func void @test_builtin_iaddcarrycc(i8 %a, i8 %b) { + entry: + %0 = alloca %i8struct + call void @_Z17__spirv_IAddCarrycc(ptr sret (%i8struct) %0, i8 %a, i8 %b) + ret void +} +; CHECK-SPIRV: [[a:%[a-z0-9_]+]] = OpFunctionParameter [[uchar]] +; CHECK-SPIRV: [[b:%[a-z0-9_]+]] = OpFunctionParameter [[uchar]] +; CHECK-SPIRV: [[entry:%[a-z0-9_]+]] = OpLabel +; CHECK-SPIRV: [[var_11:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i8struct]] Function +; CHECK-SPIRV: [[var_12:%[a-z0-9_]+]] = OpIAddCarry [[i8struct]] [[a]] [[b]] +; CHECK-SPIRV: OpStore [[var_11]] [[var_12]] +; CHECK-SPIRV: OpReturn +; CHECK-SPIRV: OpFunctionEnd + +define spir_func void @test_builtin_iaddcarryss(i16 %a, i16 %b) { + entry: + %0 = alloca %i16struct + call void @_Z17__spirv_IAddCarryss(ptr sret (%i16struct) %0, i16 %a, i16 %b) + ret void +} +; CHECK-SPIRV: [[a_0:%[a-z0-9_]+]] = OpFunctionParameter [[ushort]] +; CHECK-SPIRV: [[b_0:%[a-z0-9_]+]] = OpFunctionParameter [[ushort]] +; CHECK-SPIRV: [[entry_0:%[a-z0-9_]+]] = OpLabel +; CHECK-SPIRV: [[var_21:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i16struct]] Function +; CHECK-SPIRV: [[var_22:%[a-z0-9_]+]] = OpIAddCarry [[i16struct]] [[a_0]] [[b_0]] +; CHECK-SPIRV: OpStore [[var_21]] [[var_22]] +; CHECK-SPIRV: OpReturn +; CHECK-SPIRV: OpFunctionEnd + +define spir_func void @test_builtin_iaddcarryii(i32 %a, i32 %b) { + entry: + %0 = alloca %i32struct + call void @_Z17__spirv_IAddCarryii(ptr sret (%i32struct) %0, i32 %a, i32 %b) + ret void +} +; CHECK-SPIRV: [[a_1:%[a-z0-9_]+]] = OpFunctionParameter [[uint]] +; CHECK-SPIRV: [[b_1:%[a-z0-9_]+]] = OpFunctionParameter [[uint]] +; CHECK-SPIRV: [[entry_1:%[a-z0-9_]+]] = OpLabel +; CHECK-SPIRV: [[var_31:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i32struct]] Function +; CHECK-SPIRV: [[var_32:%[a-z0-9_]+]] = OpIAddCarry [[i32struct]] [[a_1]] [[b_1]] +; CHECK-SPIRV: OpStore [[var_31]] [[var_32]] +; CHECK-SPIRV: OpReturn +; CHECK-SPIRV: OpFunctionEnd + +define spir_func void @test_builtin_iaddcarryll(i64 %a, i64 %b) { + entry: + %0 = alloca %i64struct + call void @_Z17__spirv_IAddCarryll(ptr sret (%i64struct) %0, i64 %a, i64 %b) + ret void +} +; CHECK-SPIRV: [[a_2:%[a-z0-9_]+]] = OpFunctionParameter [[ulong]] +; CHECK-SPIRV: [[b_2:%[a-z0-9_]+]] = OpFunctionParameter [[ulong]] +; CHECK-SPIRV: [[entry_2:%[a-z0-9_]+]] = OpLabel +; CHECK-SPIRV: [[var_41:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i64struct]] Function +; CHECK-SPIRV: [[var_42:%[a-z0-9_]+]] = OpIAddCarry [[i64struct]] [[a_2]] [[b_2]] +; CHECK-SPIRV: OpStore [[var_41]] [[var_42]] +; CHECK-SPIRV: OpReturn +; CHECK-SPIRV: OpFunctionEnd + +define spir_func void @test_builtin_iaddcarryDv4_xS_(<4 x i32> %a, <4 x i32> %b) { + entry: + %0 = alloca %vecstruct + call void @_Z17__spirv_IAddCarryDv4_iS_(ptr sret (%vecstruct) %0, <4 x i32> %a, <4 x i32> %b) + ret void +} +; CHECK-SPIRV: [[a_3:%[a-z0-9_]+]] = OpFunctionParameter [[v4uint]] +; CHECK-SPIRV: [[b_3:%[a-z0-9_]+]] = OpFunctionParameter [[v4uint]] +; CHECK-SPIRV: [[entry_3:%[a-z0-9_]+]] = OpLabel +; CHECK-SPIRV: [[var_51:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_vecstruct]] Function +; CHECK-SPIRV: [[var_52:%[a-z0-9_]+]] = OpIAddCarry [[vecstruct]] [[a_3]] [[b_3]] +; CHECK-SPIRV: OpStore [[var_51]] [[var_52]] +; CHECK-SPIRV: OpReturn +; CHECK-SPIRV: OpFunctionEnd + +%struct.anon = type { i32, i32 } + +define spir_func void @test_builtin_iaddcarry_anon(i32 %a, i32 %b) { + entry: + %0 = alloca %struct.anon + %1 = addrspacecast ptr %0 to ptr addrspace(4) + call spir_func void @_Z17__spirv_IAddCarryIiiE4anonIT_T0_ES1_S2_(ptr addrspace(4) sret(%struct.anon) align 4 %1, i32 %a, i32 %b) + ret void +} +; CHECK-SPIRV: [[a_4:%[a-z0-9_]+]] = OpFunctionParameter [[uint]] +; CHECK-SPIRV: [[b_4:%[a-z0-9_]+]] = OpFunctionParameter [[uint]] +; CHECK-SPIRV: [[entry_4:%[a-z0-9_]+]] = OpLabel +; CHECK-SPIRV: [[var_59:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_struct_anon]] Function +; CHECK-SPIRV: [[var_61:%[a-z0-9_]+]] = OpPtrCastToGeneric [[_ptr_Generic_struct_anon]] [[var_59]] +; CHECK-SPIRV: [[var_62:%[a-z0-9_]+]] = OpIAddCarry [[struct_anon]] [[a_4]] [[b_4]] +; CHECK-SPIRV: OpStore [[var_61]] [[var_62]] + +declare void @_Z17__spirv_IAddCarryIiiE4anonIT_T0_ES1_S2_(ptr addrspace(4) sret(%struct.anon) align 4, i32, i32) +declare void @_Z17__spirv_IAddCarrycc(ptr sret(%i8struct), i8, i8) +declare void @_Z17__spirv_IAddCarryss(ptr sret(%i16struct), i16, i16) +declare void @_Z17__spirv_IAddCarryii(ptr sret(%i32struct), i32, i32) +declare void @_Z17__spirv_IAddCarryll(ptr sret(%i64struct), i64, i64) +declare void @_Z17__spirv_IAddCarryDv4_iS_(ptr sret (%vecstruct), <4 x i32>, <4 x i32>) diff --git a/llvm/test/CodeGen/SPIRV/instructions/scalar-integer-arithmetic.ll b/llvm/test/CodeGen/SPIRV/instructions/scalar-integer-arithmetic.ll index d222dfa570cf7..411866a67cc67 100644 --- a/llvm/test/CodeGen/SPIRV/instructions/scalar-integer-arithmetic.ll +++ b/llvm/test/CodeGen/SPIRV/instructions/scalar-integer-arithmetic.ll @@ -1,5 +1,11 @@ +; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} + ; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %} +; CHECK-DAG: OpName [[BOOL_ADD:%.+]] "bool_add" +; CHECK-DAG: OpName [[BOOL_SUB:%.+]] "bool_sub" ; CHECK-DAG: OpName [[SCALAR_ADD:%.+]] "scalar_add" ; CHECK-DAG: OpName [[SCALAR_SUB:%.+]] "scalar_sub" ; CHECK-DAG: OpName [[SCALAR_MUL:%.+]] "scalar_mul" @@ -10,13 +16,28 @@ ; CHECK-NOT: DAG-FENCE +; CHECK-DAG: [[BOOL:%.+]] = OpTypeBool ; CHECK-DAG: [[SCALAR:%.+]] = OpTypeInt 32 ; CHECK-DAG: [[SCALAR_FN:%.+]] = OpTypeFunction [[SCALAR]] [[SCALAR]] [[SCALAR]] +; CHECK-DAG: [[BOOL_FN:%.+]] = OpTypeFunction [[BOOL]] [[BOOL]] [[BOOL]] ; CHECK-NOT: DAG-FENCE ;; Test add on scalar: +define i1 @bool_add(i1 %a, i1 %b) { + %c = add i1 %a, %b + ret i1 %c +} + +; CHECK: [[BOOL_ADD]] = OpFunction [[BOOL]] None [[BOOL_FN]] +; CHECK-NEXT: [[A:%.+]] = OpFunctionParameter [[BOOL]] +; CHECK-NEXT: [[B:%.+]] = OpFunctionParameter [[BOOL]] +; CHECK: OpLabel +; CHECK: [[C:%.+]] = OpLogicalNotEqual [[BOOL]] [[A]] [[B]] +; CHECK: OpReturnValue [[C]] +; CHECK-NEXT: OpFunctionEnd + define i32 @scalar_add(i32 %a, i32 %b) { %c = add i32 %a, %b ret i32 %c @@ -32,6 +53,19 @@ define i32 @scalar_add(i32 %a, i32 %b) { ;; Test sub on scalar: +define i1 @bool_sub(i1 %a, i1 %b) { + %c = sub i1 %a, %b + ret i1 %c +} + +; CHECK: [[BOOL_SUB]] = OpFunction [[BOOL]] None [[BOOL_FN]] +; CHECK-NEXT: [[A:%.+]] = OpFunctionParameter [[BOOL]] +; CHECK-NEXT: [[B:%.+]] = OpFunctionParameter [[BOOL]] +; CHECK: OpLabel +; CHECK: [[C:%.+]] = OpLogicalNotEqual [[BOOL]] [[A]] [[B]] +; CHECK: OpReturnValue [[C]] +; CHECK-NEXT: OpFunctionEnd + define i32 @scalar_sub(i32 %a, i32 %b) { %c = sub i32 %a, %b ret i32 %c diff --git a/llvm/test/CodeGen/SPIRV/isubborrow-builtin.ll b/llvm/test/CodeGen/SPIRV/isubborrow-builtin.ll new file mode 100644 index 0000000000000..08b4d2a1fa8e5 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/isubborrow-builtin.ll @@ -0,0 +1,127 @@ +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +%i8struct = type {i8, i8} +%i16struct = type {i16, i16} +%i32struct = type {i32, i32} +%i64struct = type {i64, i64} +%vecstruct = type {<4 x i32>, <4 x i32>} + +; CHECK-SPIRV-DAG: [[uchar:%[a-z0-9_]+]] = OpTypeInt 8 +; CHECK-SPIRV-DAG: [[ushort:%[a-z0-9_]+]] = OpTypeInt 16 +; CHECK-SPIRV-DAG: [[uint:%[a-z0-9_]+]] = OpTypeInt 32 +; CHECK-SPIRV-DAG: [[ulong:%[a-z0-9_]+]] = OpTypeInt 64 +; CHECK-SPIRV-DAG: [[void:%[a-z0-9_]+]] = OpTypeVoid +; CHECK-SPIRV-DAG: [[i8struct:%[a-z0-9_]+]] = OpTypeStruct [[uchar]] [[uchar]] +; CHECK-SPIRV-DAG: [[_ptr_Function_i8struct:%[a-z0-9_]+]] = OpTypePointer Function [[i8struct]] +; CHECK-SPIRV-DAG: [[i16struct:%[a-z0-9_]+]] = OpTypeStruct [[ushort]] [[ushort]] +; CHECK-SPIRV-DAG: [[_ptr_Function_i16struct:%[a-z0-9_]+]] = OpTypePointer Function [[i16struct]] +; CHECK-SPIRV-DAG: [[i32struct:%[a-z0-9_]+]] = OpTypeStruct [[uint]] [[uint]] +; CHECK-SPIRV-DAG: [[_ptr_Function_i32struct:%[a-z0-9_]+]] = OpTypePointer Function [[i32struct]] +; CHECK-SPIRV-DAG: [[i64struct:%[a-z0-9_]+]] = OpTypeStruct [[ulong]] [[ulong]] +; CHECK-SPIRV-DAG: [[_ptr_Function_i64struct:%[a-z0-9_]+]] = OpTypePointer Function [[i64struct]] +; CHECK-SPIRV-DAG: [[v4uint:%[a-z0-9_]+]] = OpTypeVector [[uint]] 4 +; CHECK-SPIRV-DAG: [[vecstruct:%[a-z0-9_]+]] = OpTypeStruct [[v4uint]] [[v4uint]] +; CHECK-SPIRV-DAG: [[_ptr_Function_vecstruct:%[a-z0-9_]+]] = OpTypePointer Function [[vecstruct]] +; CHECK-SPIRV-DAG: [[struct_anon:%[a-z0-9_.]+]] = OpTypeStruct [[uint]] [[uint]] +; CHECK-SPIRV-DAG: [[_ptr_Function_struct_anon:%[a-z0-9_]+]] = OpTypePointer Function [[struct_anon]] +; CHECK-SPIRV-DAG: [[_ptr_Generic_struct_anon:%[a-z0-9_]+]] = OpTypePointer Generic [[struct_anon]] + +define spir_func void @test_builtin_isubborrowcc(i8 %a, i8 %b) { + entry: + %0 = alloca %i8struct + call void @_Z18__spirv_ISubBorrowcc(ptr sret (%i8struct) %0, i8 %a, i8 %b) + ret void +} +; CHECK-SPIRV: [[a:%[a-z0-9_]+]] = OpFunctionParameter [[uchar]] +; CHECK-SPIRV: [[b:%[a-z0-9_]+]] = OpFunctionParameter [[uchar]] +; CHECK-SPIRV: [[entry:%[a-z0-9_]+]] = OpLabel +; CHECK-SPIRV: [[var_11:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i8struct]] Function +; CHECK-SPIRV: [[var_12:%[a-z0-9_]+]] = OpISubBorrow [[i8struct]] [[a]] [[b]] +; CHECK-SPIRV: OpStore [[var_11]] [[var_12]] +; CHECK-SPIRV: OpReturn +; CHECK-SPIRV: OpFunctionEnd + +define spir_func void @test_builtin_isubborrowss(i16 %a, i16 %b) { + entry: + %0 = alloca %i16struct + call void @_Z18__spirv_ISubBorrowss(ptr sret (%i16struct) %0, i16 %a, i16 %b) + ret void +} +; CHECK-SPIRV: [[a_0:%[a-z0-9_]+]] = OpFunctionParameter [[ushort]] +; CHECK-SPIRV: [[b_0:%[a-z0-9_]+]] = OpFunctionParameter [[ushort]] +; CHECK-SPIRV: [[entry_0:%[a-z0-9_]+]] = OpLabel +; CHECK-SPIRV: [[var_21:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i16struct]] Function +; CHECK-SPIRV: [[var_22:%[a-z0-9_]+]] = OpISubBorrow [[i16struct]] [[a_0]] [[b_0]] +; CHECK-SPIRV: OpStore [[var_21]] [[var_22]] +; CHECK-SPIRV: OpReturn +; CHECK-SPIRV: OpFunctionEnd + +define spir_func void @test_builtin_isubborrowii(i32 %a, i32 %b) { + entry: + %0 = alloca %i32struct + call void @_Z18__spirv_ISubBorrowii(ptr sret (%i32struct) %0, i32 %a, i32 %b) + ret void +} +; CHECK-SPIRV: [[a_1:%[a-z0-9_]+]] = OpFunctionParameter [[uint]] +; CHECK-SPIRV: [[b_1:%[a-z0-9_]+]] = OpFunctionParameter [[uint]] +; CHECK-SPIRV: [[entry_1:%[a-z0-9_]+]] = OpLabel +; CHECK-SPIRV: [[var_31:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i32struct]] Function +; CHECK-SPIRV: [[var_32:%[a-z0-9_]+]] = OpISubBorrow [[i32struct]] [[a_1]] [[b_1]] +; CHECK-SPIRV: OpStore [[var_31]] [[var_32]] +; CHECK-SPIRV: OpReturn +; CHECK-SPIRV: OpFunctionEnd + +define spir_func void @test_builtin_isubborrowll(i64 %a, i64 %b) { + entry: + %0 = alloca %i64struct + call void @_Z18__spirv_ISubBorrowll(ptr sret (%i64struct) %0, i64 %a, i64 %b) + ret void +} +; CHECK-SPIRV: [[a_2:%[a-z0-9_]+]] = OpFunctionParameter [[ulong]] +; CHECK-SPIRV: [[b_2:%[a-z0-9_]+]] = OpFunctionParameter [[ulong]] +; CHECK-SPIRV: [[entry_2:%[a-z0-9_]+]] = OpLabel +; CHECK-SPIRV: [[var_41:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_i64struct]] Function +; CHECK-SPIRV: [[var_42:%[a-z0-9_]+]] = OpISubBorrow [[i64struct]] [[a_2]] [[b_2]] +; CHECK-SPIRV: OpStore [[var_41]] [[var_42]] +; CHECK-SPIRV: OpReturn +; CHECK-SPIRV: OpFunctionEnd + +define spir_func void @test_builtin_isubborrowDv4_xS_(<4 x i32> %a, <4 x i32> %b) { + entry: + %0 = alloca %vecstruct + call void @_Z18__spirv_ISubBorrowDv4_iS_(ptr sret (%vecstruct) %0, <4 x i32> %a, <4 x i32> %b) + ret void +} +; CHECK-SPIRV: [[a_3:%[a-z0-9_]+]] = OpFunctionParameter [[v4uint]] +; CHECK-SPIRV: [[b_3:%[a-z0-9_]+]] = OpFunctionParameter [[v4uint]] +; CHECK-SPIRV: [[entry_3:%[a-z0-9_]+]] = OpLabel +; CHECK-SPIRV: [[var_51:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_vecstruct]] Function +; CHECK-SPIRV: [[var_52:%[a-z0-9_]+]] = OpISubBorrow [[vecstruct]] [[a_3]] [[b_3]] +; CHECK-SPIRV: OpStore [[var_51]] [[var_52]] +; CHECK-SPIRV: OpReturn +; CHECK-SPIRV: OpFunctionEnd + +%struct.anon = type { i32, i32 } + +define spir_func void @test_builtin_isubborrow_anon(i32 %a, i32 %b) { + entry: + %0 = alloca %struct.anon + %1 = addrspacecast ptr %0 to ptr addrspace(4) + call spir_func void @_Z18__spirv_ISubBorrowIiiE4anonIT_T0_ES1_S2_(ptr addrspace(4) sret(%struct.anon) align 4 %1, i32 %a, i32 %b) + ret void +} +; CHECK-SPIRV: [[a_4:%[a-z0-9_]+]] = OpFunctionParameter [[uint]] +; CHECK-SPIRV: [[b_4:%[a-z0-9_]+]] = OpFunctionParameter [[uint]] +; CHECK-SPIRV: [[entry_4:%[a-z0-9_]+]] = OpLabel +; CHECK-SPIRV: [[var_59:%[a-z0-9_]+]] = OpVariable [[_ptr_Function_struct_anon]] Function +; CHECK-SPIRV: [[var_61:%[a-z0-9_]+]] = OpPtrCastToGeneric [[_ptr_Generic_struct_anon]] [[var_59]] +; CHECK-SPIRV: [[var_62:%[a-z0-9_]+]] = OpISubBorrow [[struct_anon]] [[a_4]] [[b_4]] +; CHECK-SPIRV: OpStore [[var_61]] [[var_62]] + +declare void @_Z18__spirv_ISubBorrowIiiE4anonIT_T0_ES1_S2_(ptr addrspace(4) sret(%struct.anon) align 4, i32, i32) +declare void @_Z18__spirv_ISubBorrowcc(ptr sret(%i8struct), i8, i8) +declare void @_Z18__spirv_ISubBorrowss(ptr sret(%i16struct), i16, i16) +declare void @_Z18__spirv_ISubBorrowii(ptr sret(%i32struct), i32, i32) +declare void @_Z18__spirv_ISubBorrowll(ptr sret(%i64struct), i64, i64) +declare void @_Z18__spirv_ISubBorrowDv4_iS_(ptr sret (%vecstruct), <4 x i32>, <4 x i32>) diff --git a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/uadd.with.overflow.ll b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/uadd.with.overflow.ll index cecd6f60655dc..1c77800a1630d 100644 --- a/llvm/test/CodeGen/SPIRV/llvm-intrinsics/uadd.with.overflow.ll +++ b/llvm/test/CodeGen/SPIRV/llvm-intrinsics/uadd.with.overflow.ll @@ -20,6 +20,7 @@ ; CHECK-DAG: %[[PtrV2Long:.*]] = OpTypePointer Function %[[V2Long]] ; CHECK-DAG: %[[StructV2Long:.*]] = OpTypeStruct %[[V2Long]] %[[V2Long]] ; CHECK-DAG: %[[ZeroV2Long:.*]] = OpConstantNull %[[V2Long]] +; CHECK-DAG: %[[StructLong:.*]] = OpTypeStruct %[[Long]] %[[Long]] ; CHECK: OpFunction ; CHECK: %[[A:.*]] = OpFunctionParameter %[[Char]] @@ -84,6 +85,20 @@ define dso_local spir_func void @umulo_v2i64(<2 x i64> %a, <2 x i64> %b, ptr %p) ret void } +; This is to check that we re-use the same OpTypeStruct for two identical references to { i64, i1 }. +; CHECK: OpFunction +; CHECK: OpIAddCarry %[[StructLong]] +; CHECK: OpIAddCarry %[[StructLong]] +; CHECK: OpReturn +define void @foo(i64 %a, i64 %b) { + %r1 = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 %a, i64 %b) + %r2 = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 %a, i64 %b) + %d1 = extractvalue { i64, i1 } %r1, 0 + %d2 = extractvalue { i64, i1 } %r2, 0 + ret void +} + declare {i8, i1} @llvm.uadd.with.overflow.i8(i8, i8) declare {i32, i1} @llvm.uadd.with.overflow.i32(i32, i32) +declare {i64, i1} @llvm.uadd.with.overflow.i64(i64, i64) declare {<2 x i64>, <2 x i1>} @llvm.uadd.with.overflow.v2i64(<2 x i64>, <2 x i64>) diff --git a/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering-unwrapped.ll b/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering-unwrapped.ll new file mode 100644 index 0000000000000..09c0a92d596ce --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering-unwrapped.ll @@ -0,0 +1,50 @@ +; The goal of the test case is to ensure that correct types are applied to virtual registers +; which were used as arguments in call lowering and so caused early definition of SPIR-V types. + +; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +%t_id = type { %t_arr } +%t_arr = type { [1 x i64] } +%t_bf16 = type { i16 } + +define weak_odr dso_local spir_kernel void @foo(ptr addrspace(1) align 4 %_arg_ERR, ptr byval(%t_id) align 8 %_arg_ERR3) { +entry: + %FloatArray.i = alloca [4 x float], align 4 + %BF16Array.i = alloca [4 x %t_bf16], align 2 + %0 = load i64, ptr %_arg_ERR3, align 8 + %add.ptr.i = getelementptr inbounds i32, ptr addrspace(1) %_arg_ERR, i64 %0 + %FloatArray.ascast.i = addrspacecast ptr %FloatArray.i to ptr addrspace(4) + %BF16Array.ascast.i = addrspacecast ptr %BF16Array.i to ptr addrspace(4) + call spir_func void @__devicelib_ConvertFToBF16INTELVec4(ptr addrspace(4) %FloatArray.ascast.i, ptr addrspace(4) %BF16Array.ascast.i) + br label %for.cond.i + +for.cond.i: ; preds = %for.inc.i, %entry + %lsr.iv1 = phi ptr [ %scevgep2, %for.inc.i ], [ %FloatArray.i, %entry ] + %lsr.iv = phi ptr addrspace(4) [ %scevgep, %for.inc.i ], [ %BF16Array.ascast.i, %entry ] + %i.0.i = phi i32 [ 0, %entry ], [ %inc.i, %for.inc.i ] + %cmp.i = icmp ult i32 %i.0.i, 4 + br i1 %cmp.i, label %for.body.i, label %exit + +for.body.i: ; preds = %for.cond.i + %1 = load float, ptr %lsr.iv1, align 4 + %call.i.i = call spir_func float @__devicelib_ConvertBF16ToFINTEL(ptr addrspace(4) align 2 dereferenceable(2) %lsr.iv) + %cmp5.i = fcmp une float %1, %call.i.i + br i1 %cmp5.i, label %if.then.i, label %for.inc.i + +if.then.i: ; preds = %for.body.i + store i32 1, ptr addrspace(1) %add.ptr.i, align 4 + br label %for.inc.i + +for.inc.i: ; preds = %if.then.i, %for.body.i + %inc.i = add nuw nsw i32 %i.0.i, 1 + %scevgep = getelementptr i8, ptr addrspace(4) %lsr.iv, i64 2 + %scevgep2 = getelementptr i8, ptr %lsr.iv1, i64 4 + br label %for.cond.i + +exit: ; preds = %for.cond.i + ret void +} + +declare void @llvm.memcpy.p0.p1.i64(ptr noalias nocapture writeonly, ptr addrspace(1) noalias nocapture readonly, i64, i1 immarg) +declare dso_local spir_func void @__devicelib_ConvertFToBF16INTELVec4(ptr addrspace(4), ptr addrspace(4)) +declare dso_local spir_func float @__devicelib_ConvertBF16ToFINTEL(ptr addrspace(4) align 2 dereferenceable(2)) diff --git a/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering.ll b/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering.ll new file mode 100644 index 0000000000000..a31638e0b8704 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/pointers/phi-valid-operand-types-vs-calllowering.ll @@ -0,0 +1,49 @@ +; The goal of the test case is to ensure that correct types are applied to virtual registers +; which were used as arguments in call lowering and so caused early definition of SPIR-V types. + +; RUN: %if spirv-tools %{ llc -O2 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %} + +%t_id = type { %t_arr } +%t_arr = type { [1 x i64] } +%t_bf16 = type { i16 } + +define weak_odr dso_local spir_kernel void @foo(ptr addrspace(1) align 4 %_arg_ERR, ptr byval(%t_id) align 8 %_arg_ERR3) { +entry: + %FloatArray.i = alloca [4 x float], align 4 + %BF16Array.i = alloca [4 x %t_bf16], align 2 + %0 = load i64, ptr %_arg_ERR3, align 8 + %add.ptr.i = getelementptr inbounds i32, ptr addrspace(1) %_arg_ERR, i64 %0 + %FloatArray.ascast.i = addrspacecast ptr %FloatArray.i to ptr addrspace(4) + %BF16Array.ascast.i = addrspacecast ptr %BF16Array.i to ptr addrspace(4) + call spir_func void @__devicelib_ConvertFToBF16INTELVec4(ptr addrspace(4) %FloatArray.ascast.i, ptr addrspace(4) %BF16Array.ascast.i) + br label %for.cond.i + +for.cond.i: ; preds = %for.inc.i, %entry + %i.0.i = phi i32 [ 0, %entry ], [ %inc.i, %for.inc.i ] + %cmp.i = icmp ult i32 %i.0.i, 4 + br i1 %cmp.i, label %for.body.i, label %exit + +for.body.i: ; preds = %for.cond.i + %idxprom.i = zext nneg i32 %i.0.i to i64 + %arrayidx.i = getelementptr inbounds [4 x float], ptr %FloatArray.i, i64 0, i64 %idxprom.i + %1 = load float, ptr %arrayidx.i, align 4 + %arrayidx4.i = getelementptr inbounds [4 x %t_bf16], ptr addrspace(4) %BF16Array.ascast.i, i64 0, i64 %idxprom.i + %call.i.i = call spir_func float @__devicelib_ConvertBF16ToFINTEL(ptr addrspace(4) align 2 dereferenceable(2) %arrayidx4.i) + %cmp5.i = fcmp une float %1, %call.i.i + br i1 %cmp5.i, label %if.then.i, label %for.inc.i + +if.then.i: ; preds = %for.body.i + store i32 1, ptr addrspace(1) %add.ptr.i, align 4 + br label %for.inc.i + +for.inc.i: ; preds = %if.then.i, %for.body.i + %inc.i = add nuw nsw i32 %i.0.i, 1 + br label %for.cond.i + +exit: ; preds = %for.cond.i + ret void +} + +declare void @llvm.memcpy.p0.p1.i64(ptr noalias nocapture writeonly, ptr addrspace(1) noalias nocapture readonly, i64, i1 immarg) +declare dso_local spir_func void @__devicelib_ConvertFToBF16INTELVec4(ptr addrspace(4), ptr addrspace(4)) +declare dso_local spir_func float @__devicelib_ConvertBF16ToFINTEL(ptr addrspace(4) align 2 dereferenceable(2))