diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp index e090fb67b3231..78c0d252cb834 100644 --- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp @@ -1656,10 +1656,29 @@ static bool generateBarrierInst(const SPIRV::IncomingCall *Call, } static bool generateCastToPtrInst(const SPIRV::IncomingCall *Call, - MachineIRBuilder &MIRBuilder) { - MIRBuilder.buildInstr(TargetOpcode::G_ADDRSPACE_CAST) - .addDef(Call->ReturnRegister) - .addUse(Call->Arguments[0]); + MachineIRBuilder &MIRBuilder, + SPIRVGlobalRegistry *GR) { + // Lookup the instruction opcode in the TableGen records. + const SPIRV::DemangledBuiltin *Builtin = Call->Builtin; + unsigned Opcode = + SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode; + + if (Opcode == SPIRV::OpGenericCastToPtrExplicit) { + SPIRV::StorageClass::StorageClass ResSC = + GR->getPointerStorageClass(Call->ReturnRegister); + if (!isGenericCastablePtr(ResSC)) + return false; + + MIRBuilder.buildInstr(Opcode) + .addDef(Call->ReturnRegister) + .addUse(GR->getSPIRVTypeID(Call->ReturnType)) + .addUse(Call->Arguments[0]) + .addImm(ResSC); + } else { + MIRBuilder.buildInstr(TargetOpcode::G_ADDRSPACE_CAST) + .addDef(Call->ReturnRegister) + .addUse(Call->Arguments[0]); + } return true; } @@ -2833,7 +2852,7 @@ std::optional lowerBuiltin(const StringRef DemangledCall, case SPIRV::Barrier: return generateBarrierInst(Call.get(), MIRBuilder, GR); case SPIRV::CastToPtr: - return generateCastToPtrInst(Call.get(), MIRBuilder); + return generateCastToPtrInst(Call.get(), MIRBuilder, GR); case SPIRV::Dot: case SPIRV::IntegerDot: return generateDotOrFMulInst(DemangledCall, Call.get(), MIRBuilder, GR); diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td index a3f27dde76b65..59cd38126cc01 100644 --- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td +++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td @@ -687,15 +687,15 @@ defm : DemangledNativeBuiltin<"__spirv_Load", OpenCL_std, LoadStore, 1, 3, OpLoa defm : DemangledNativeBuiltin<"__spirv_Store", OpenCL_std, LoadStore, 2, 4, OpStore>; // Address Space Qualifier Functions/Pointers Conversion Instructions: -defm : DemangledNativeBuiltin<"to_global", OpenCL_std, CastToPtr, 1, 1, OpGenericCastToPtr>; -defm : DemangledNativeBuiltin<"to_local", OpenCL_std, CastToPtr, 1, 1, OpGenericCastToPtr>; -defm : DemangledNativeBuiltin<"to_private", OpenCL_std, CastToPtr, 1, 1, OpGenericCastToPtr>; +defm : DemangledNativeBuiltin<"to_global", OpenCL_std, CastToPtr, 1, 1, OpGenericCastToPtrExplicit>; +defm : DemangledNativeBuiltin<"to_local", OpenCL_std, CastToPtr, 1, 1, OpGenericCastToPtrExplicit>; +defm : DemangledNativeBuiltin<"to_private", OpenCL_std, CastToPtr, 1, 1, OpGenericCastToPtrExplicit>; defm : DemangledNativeBuiltin<"__spirv_GenericCastToPtr_ToGlobal", OpenCL_std, CastToPtr, 2, 2, OpGenericCastToPtr>; defm : DemangledNativeBuiltin<"__spirv_GenericCastToPtr_ToLocal", OpenCL_std, CastToPtr, 2, 2, OpGenericCastToPtr>; defm : DemangledNativeBuiltin<"__spirv_GenericCastToPtr_ToPrivate", OpenCL_std, CastToPtr, 2, 2, OpGenericCastToPtr>; -defm : DemangledNativeBuiltin<"__spirv_GenericCastToPtrExplicit_ToGlobal", OpenCL_std, CastToPtr, 2, 2, OpGenericCastToPtr>; -defm : DemangledNativeBuiltin<"__spirv_GenericCastToPtrExplicit_ToLocal", OpenCL_std, CastToPtr, 2, 2, OpGenericCastToPtr>; -defm : DemangledNativeBuiltin<"__spirv_GenericCastToPtrExplicit_ToPrivate", OpenCL_std, CastToPtr, 2, 2, OpGenericCastToPtr>; +defm : DemangledNativeBuiltin<"__spirv_GenericCastToPtrExplicit_ToGlobal", OpenCL_std, CastToPtr, 2, 2, OpGenericCastToPtrExplicit>; +defm : DemangledNativeBuiltin<"__spirv_GenericCastToPtrExplicit_ToLocal", OpenCL_std, CastToPtr, 2, 2, OpGenericCastToPtrExplicit>; +defm : DemangledNativeBuiltin<"__spirv_GenericCastToPtrExplicit_ToPrivate", OpenCL_std, CastToPtr, 2, 2, OpGenericCastToPtrExplicit>; // Cooperative Matrix builtin records: defm : DemangledNativeBuiltin<"__spirv_CooperativeMatrixLoadKHR", OpenCL_std, CoopMatr, 2, 4, OpCooperativeMatrixLoadKHR>; diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp index d274839af82eb..216c3e26be1bf 100644 --- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp @@ -390,6 +390,7 @@ void SPIRVTargetLowering::finalizeLowering(MachineFunction &MF) const { break; case SPIRV::OpPtrCastToGeneric: case SPIRV::OpGenericCastToPtr: + case SPIRV::OpGenericCastToPtrExplicit: validateAccessChain(STI, MRI, GR, MI); break; case SPIRV::OpPtrAccessChain: diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp index 79f6b43f3aded..47f8ace4f7d0b 100644 --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -1552,17 +1552,6 @@ bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg, .constrainAllUses(TII, TRI, RBI); } -static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) { - switch (SC) { - case SPIRV::StorageClass::Workgroup: - case SPIRV::StorageClass::CrossWorkgroup: - case SPIRV::StorageClass::Function: - return true; - default: - return false; - } -} - static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) { switch (SC) { case SPIRV::StorageClass::DeviceOnlyINTEL: diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td index afd3a5206926c..cc32200a0a261 100644 --- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td +++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td @@ -1683,6 +1683,7 @@ defm InBoundsAccessChain : OpcodeOperand<66>; defm InBoundsPtrAccessChain : OpcodeOperand<70>; defm PtrCastToGeneric : OpcodeOperand<121>; defm GenericCastToPtr : OpcodeOperand<122>; +defm GenericCastToPtrExplicit : OpcodeOperand<123>; defm Bitcast : OpcodeOperand<124>; defm ConvertPtrToU : OpcodeOperand<117>; defm ConvertUToPtr : OpcodeOperand<120>; diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.h b/llvm/lib/Target/SPIRV/SPIRVUtils.h index 0498c7beb073c..ccf394de45c89 100644 --- a/llvm/lib/Target/SPIRV/SPIRVUtils.h +++ b/llvm/lib/Target/SPIRV/SPIRVUtils.h @@ -167,6 +167,19 @@ MachineBasicBlock::iterator getOpVariableMBBIt(MachineInstr &I); // terminators and debug instructions. MachineBasicBlock::iterator getInsertPtValidEnd(MachineBasicBlock *MBB); +// Returns true if a pointer to the storage class can be casted to/from a +// pointer to the Generic storage class. +constexpr bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) { + switch (SC) { + case SPIRV::StorageClass::Workgroup: + case SPIRV::StorageClass::CrossWorkgroup: + case SPIRV::StorageClass::Function: + return true; + default: + return false; + } +} + // Convert a SPIR-V storage class to the corresponding LLVM IR address space. // TODO: maybe the following two functions should be handled in the subtarget // to allow for different OpenCL vs Vulkan handling. diff --git a/llvm/test/CodeGen/SPIRV/transcoding/OpGenericCastToPtr.ll b/llvm/test/CodeGen/SPIRV/transcoding/OpGenericCastToPtr.ll index 2cba0f6ebd74b..119dbe14446c1 100644 --- a/llvm/test/CodeGen/SPIRV/transcoding/OpGenericCastToPtr.ll +++ b/llvm/test/CodeGen/SPIRV/transcoding/OpGenericCastToPtr.ll @@ -24,9 +24,9 @@ ; CHECK-SPIRV-NEXT: OpGenericCastToPtr %[[#GlobalIntPtr]] ; CHECK-SPIRV-NEXT: OpGenericCastToPtr %[[#LocalCharPtr]] ; CHECK-SPIRV-NEXT: OpGenericCastToPtr %[[#PrivateIntPtr]] -; CHECK-SPIRV-NEXT: OpGenericCastToPtr %[[#GlobalIntPtr]] -; CHECK-SPIRV-NEXT: OpGenericCastToPtr %[[#LocalCharPtr]] -; CHECK-SPIRV-NEXT: OpGenericCastToPtr %[[#PrivateIntPtr]] +; CHECK-SPIRV-NEXT: OpGenericCastToPtrExplicit %[[#GlobalIntPtr]] %{{.*}} CrossWorkgroup +; CHECK-SPIRV-NEXT: OpGenericCastToPtrExplicit %[[#LocalCharPtr]] %{{.*}} Workgroup +; CHECK-SPIRV-NEXT: OpGenericCastToPtrExplicit %[[#PrivateIntPtr]] %{{.*}} Function ; CHECK-SPIRV: OpFunctionEnd define spir_kernel void @test1(ptr addrspace(1) %_arg_GlobalA, ptr byval(%id) %_arg_GlobalId, ptr addrspace(3) %_arg_LocalA) { @@ -52,9 +52,9 @@ entry: ; CHECK-SPIRV: OpPtrCastToGeneric %[[#GenericIntPtr]] ; CHECK-SPIRV-NEXT: OpPtrCastToGeneric %[[#GenericCharPtr]] ; CHECK-SPIRV-NEXT: OpPtrCastToGeneric %[[#GenericIntPtr]] -; CHECK-SPIRV-NEXT: OpGenericCastToPtr %[[#GlobalIntPtr]] -; CHECK-SPIRV-NEXT: OpGenericCastToPtr %[[#LocalCharPtr]] -; CHECK-SPIRV-NEXT: OpGenericCastToPtr %[[#PrivateIntPtr]] +; CHECK-SPIRV-NEXT: OpGenericCastToPtrExplicit %[[#GlobalIntPtr]] %{{.*}} CrossWorkgroup +; CHECK-SPIRV-NEXT: OpGenericCastToPtrExplicit %[[#LocalCharPtr]] %{{.*}} Workgroup +; CHECK-SPIRV-NEXT: OpGenericCastToPtrExplicit %[[#PrivateIntPtr]] %{{.*}} Function ; CHECK-SPIRV: OpFunctionEnd define spir_kernel void @test2(ptr addrspace(1) %_arg_GlobalA, ptr byval(%id) %_arg_GlobalId, ptr addrspace(3) %_arg_LocalA) { @@ -93,18 +93,9 @@ declare spir_func ptr @_Z10to_privatePv(ptr addrspace(4)) ; CHECK-SPIRV-NEXT: OpGenericCastToPtr %[[#GlobalIntPtr]] ; CHECK-SPIRV-NEXT: OpGenericCastToPtr %[[#LocalCharPtr]] ; CHECK-SPIRV-NEXT: OpGenericCastToPtr %[[#PrivateIntPtr]] -; CHECK-SPIRV-NEXT: OpGenericCastToPtr %[[#GlobalIntPtr]] -; CHECK-SPIRV-NEXT: OpGenericCastToPtr %[[#LocalCharPtr]] -; CHECK-SPIRV-NEXT: OpGenericCastToPtr %[[#PrivateIntPtr]] -; CHECK-SPIRV: OpFunctionEnd - -; CHECK-SPIRV: OpFunction -; CHECK-SPIRV: OpPtrCastToGeneric %[[#GenericIntPtr]] -; CHECK-SPIRV-NEXT: OpPtrCastToGeneric %[[#GenericCharPtr]] -; CHECK-SPIRV-NEXT: OpPtrCastToGeneric %[[#GenericIntPtr]] -; CHECK-SPIRV-NEXT: OpGenericCastToPtr %[[#GlobalIntPtr]] -; CHECK-SPIRV-NEXT: OpGenericCastToPtr %[[#LocalCharPtr]] -; CHECK-SPIRV-NEXT: OpGenericCastToPtr %[[#PrivateIntPtr]] +; CHECK-SPIRV-NEXT: OpGenericCastToPtrExplicit %[[#GlobalIntPtr]] %{{.*}} CrossWorkgroup +; CHECK-SPIRV-NEXT: OpGenericCastToPtrExplicit %[[#LocalCharPtr]] %{{.*}} Workgroup +; CHECK-SPIRV-NEXT: OpGenericCastToPtrExplicit %[[#PrivateIntPtr]] %{{.*}} Function ; CHECK-SPIRV: OpFunctionEnd define spir_kernel void @test3(ptr addrspace(1) %_arg_GlobalA, ptr byval(%id) %_arg_GlobalId, ptr addrspace(3) %_arg_LocalA) { @@ -126,6 +117,15 @@ entry: ret void } +; CHECK-SPIRV: OpFunction +; CHECK-SPIRV: OpPtrCastToGeneric %[[#GenericIntPtr]] +; CHECK-SPIRV-NEXT: OpPtrCastToGeneric %[[#GenericCharPtr]] +; CHECK-SPIRV-NEXT: OpPtrCastToGeneric %[[#GenericIntPtr]] +; CHECK-SPIRV-NEXT: OpGenericCastToPtrExplicit %[[#GlobalIntPtr]] %{{.*}} CrossWorkgroup +; CHECK-SPIRV-NEXT: OpGenericCastToPtrExplicit %[[#LocalCharPtr]] %{{.*}} Workgroup +; CHECK-SPIRV-NEXT: OpGenericCastToPtrExplicit %[[#PrivateIntPtr]] %{{.*}} Function +; CHECK-SPIRV: OpFunctionEnd + define spir_kernel void @test4(ptr addrspace(1) %_arg_GlobalA, ptr byval(%id) %_arg_GlobalId, ptr addrspace(3) %_arg_LocalA) { entry: %var = alloca i32