|
17 | 17 | #include "CGObjCRuntime.h" |
18 | 18 | #include "CGOpenCLRuntime.h" |
19 | 19 | #include "CGRecordLayout.h" |
| 20 | +#include "CGValue.h" |
20 | 21 | #include "CodeGenFunction.h" |
21 | 22 | #include "CodeGenModule.h" |
22 | 23 | #include "ConstantEmitter.h" |
|
27 | 28 | #include "clang/AST/Decl.h" |
28 | 29 | #include "clang/AST/OSLog.h" |
29 | 30 | #include "clang/AST/OperationKinds.h" |
| 31 | +#include "clang/AST/Type.h" |
30 | 32 | #include "clang/Basic/TargetBuiltins.h" |
31 | 33 | #include "clang/Basic/TargetInfo.h" |
32 | 34 | #include "clang/Basic/TargetOptions.h" |
33 | 35 | #include "clang/CodeGen/CGFunctionInfo.h" |
34 | 36 | #include "clang/Frontend/FrontendDiagnostic.h" |
35 | 37 | #include "llvm/ADT/APFloat.h" |
36 | 38 | #include "llvm/ADT/APInt.h" |
37 | | -#include "llvm/ADT/ArrayRef.h" |
38 | 39 | #include "llvm/ADT/FloatingPointMode.h" |
39 | 40 | #include "llvm/ADT/SmallPtrSet.h" |
40 | | -#include "llvm/ADT/SmallVector.h" |
41 | 41 | #include "llvm/ADT/StringExtras.h" |
42 | 42 | #include "llvm/Analysis/ValueTracking.h" |
43 | 43 | #include "llvm/IR/DataLayout.h" |
44 | | -#include "llvm/IR/DerivedTypes.h" |
45 | 44 | #include "llvm/IR/InlineAsm.h" |
46 | 45 | #include "llvm/IR/Intrinsics.h" |
47 | 46 | #include "llvm/IR/IntrinsicsAArch64.h" |
|
70 | 69 | #include "llvm/TargetParser/X86TargetParser.h" |
71 | 70 | #include <optional> |
72 | 71 | #include <sstream> |
| 72 | +#include <utility> |
73 | 73 |
|
74 | 74 | using namespace clang; |
75 | 75 | using namespace CodeGen; |
@@ -18955,92 +18955,141 @@ case Builtin::BI__builtin_hlsl_elementwise_isinf: { |
18955 | 18955 | CGM.getHLSLRuntime().getRadiansIntrinsic(), ArrayRef<Value *>{Op0}, |
18956 | 18956 | nullptr, "hlsl.radians"); |
18957 | 18957 | } |
18958 | | - // This should only be called when targeting DXIL |
18959 | 18958 | case Builtin::BI__builtin_hlsl_splitdouble: { |
18960 | 18959 |
|
18961 | 18960 | assert((E->getArg(0)->getType()->hasFloatingRepresentation() && |
18962 | 18961 | E->getArg(1)->getType()->hasUnsignedIntegerRepresentation() && |
18963 | 18962 | E->getArg(2)->getType()->hasUnsignedIntegerRepresentation()) && |
18964 | 18963 | "asuint operands types mismatch"); |
18965 | 18964 | Value *Op0 = EmitScalarExpr(E->getArg(0)); |
18966 | | - const HLSLOutArgExpr *OutArg1 = dyn_cast<HLSLOutArgExpr>(E->getArg(1)); |
18967 | | - const HLSLOutArgExpr *OutArg2 = dyn_cast<HLSLOutArgExpr>(E->getArg(2)); |
| 18965 | + const auto *OutArg1 = dyn_cast<HLSLOutArgExpr>(E->getArg(1)); |
| 18966 | + const auto *OutArg2 = dyn_cast<HLSLOutArgExpr>(E->getArg(2)); |
18968 | 18967 |
|
18969 | 18968 | CallArgList Args; |
18970 | | - auto [Op1BaseLValue, Op1TmpLValue] = |
18971 | | - EmitHLSLOutArgExpr(OutArg1, Args, OutArg1->getType()); |
18972 | | - auto [Op2BaseLValue, Op2TmpLValue] = |
18973 | | - EmitHLSLOutArgExpr(OutArg2, Args, OutArg2->getType()); |
| 18969 | + LValue Op1TmpLValue = EmitHLSLOutArgExpr(OutArg1, Args, OutArg1->getType()); |
| 18970 | + LValue Op2TmpLValue = EmitHLSLOutArgExpr(OutArg2, Args, OutArg2->getType()); |
18974 | 18971 |
|
18975 | | - if (CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil) { |
| 18972 | + if (getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) |
| 18973 | + Args.reverseWritebacks(); |
18976 | 18974 |
|
18977 | | - llvm::StructType *retType = llvm::StructType::get(Int32Ty, Int32Ty); |
| 18975 | + auto EmitVectorCode = |
| 18976 | + [](Value *Op, CGBuilderTy *Builder, |
| 18977 | + FixedVectorType *DestTy) -> std::pair<Value *, Value *> { |
| 18978 | + Value *bitcast = Builder->CreateBitCast(Op, DestTy); |
18978 | 18979 |
|
18979 | | - if (Op0->getType()->isVectorTy()) { |
18980 | | - auto *Op0VecTy = E->getArg(0)->getType()->getAs<VectorType>(); |
| 18980 | + SmallVector<int> LowbitsIndex; |
| 18981 | + SmallVector<int> HighbitsIndex; |
18981 | 18982 |
|
18982 | | - llvm::VectorType *i32VecTy = llvm::VectorType::get( |
18983 | | - Int32Ty, ElementCount::getFixed(Op0VecTy->getNumElements())); |
18984 | | - retType = llvm::StructType::get(i32VecTy, i32VecTy); |
| 18983 | + for (unsigned int Idx = 0; Idx < DestTy->getNumElements(); Idx += 2) { |
| 18984 | + LowbitsIndex.push_back(Idx); |
| 18985 | + HighbitsIndex.push_back(Idx + 1); |
18985 | 18986 | } |
18986 | 18987 |
|
18987 | | - CallInst *CI = |
18988 | | - Builder.CreateIntrinsic(retType, Intrinsic::dx_splitdouble, {Op0}, |
18989 | | - nullptr, "hlsl.splitdouble"); |
18990 | | - |
18991 | | - Value *arg0 = Builder.CreateExtractValue(CI, 0); |
18992 | | - Value *arg1 = Builder.CreateExtractValue(CI, 1); |
| 18988 | + Value *Arg0 = Builder->CreateShuffleVector(bitcast, LowbitsIndex); |
| 18989 | + Value *Arg1 = Builder->CreateShuffleVector(bitcast, HighbitsIndex); |
18993 | 18990 |
|
18994 | | - Builder.CreateStore(arg0, Op1TmpLValue.getAddress()); |
18995 | | - auto *s = Builder.CreateStore(arg1, Op2TmpLValue.getAddress()); |
| 18991 | + return std::make_pair(Arg0, Arg1); |
| 18992 | + }; |
18996 | 18993 |
|
18997 | | - EmitWritebacks(*this, Args); |
18998 | | - return s; |
18999 | | - } |
| 18994 | + Value *LastInst = nullptr; |
19000 | 18995 |
|
| 18996 | + if (CGM.getTarget().getTriple().isDXIL()) { |
19001 | 18997 |
|
19002 | | - if(!Op0->getType()->isVectorTy()){ |
19003 | | - FixedVectorType *destTy = FixedVectorType::get(Int32Ty, 2); |
19004 | | - Value *bitcast = Builder.CreateBitCast(Op0, destTy); |
| 18998 | + llvm::Type *RetElementTy = Int32Ty; |
| 18999 | + if (auto *Op0VecTy = E->getArg(0)->getType()->getAs<VectorType>()) |
| 19000 | + RetElementTy = llvm::VectorType::get( |
| 19001 | + Int32Ty, ElementCount::getFixed(Op0VecTy->getNumElements())); |
| 19002 | + auto *RetTy = llvm::StructType::get(RetElementTy, RetElementTy); |
19005 | 19003 |
|
19006 | | - Value *arg0 = Builder.CreateExtractElement(bitcast, 0.0); |
19007 | | - Value *arg1 = Builder.CreateExtractElement(bitcast, 1.0); |
| 19004 | + CallInst *CI = Builder.CreateIntrinsic( |
| 19005 | + RetTy, Intrinsic::dx_splitdouble, {Op0}, nullptr, "hlsl.splitdouble"); |
19008 | 19006 |
|
19009 | | - Builder.CreateStore(arg0, Op1TmpLValue.getAddress()); |
19010 | | - auto *s = Builder.CreateStore(arg1, Op2TmpLValue.getAddress()); |
| 19007 | + Value *Arg0 = Builder.CreateExtractValue(CI, 0); |
| 19008 | + Value *Arg1 = Builder.CreateExtractValue(CI, 1); |
19011 | 19009 |
|
19012 | | - EmitWritebacks(*this, Args); |
19013 | | - return s; |
19014 | | - } |
19015 | | - |
19016 | | - auto *Op0VecTy = E->getArg(0)->getType()->getAs<VectorType>(); |
| 19010 | + Builder.CreateStore(Arg0, Op1TmpLValue.getAddress()); |
| 19011 | + LastInst = Builder.CreateStore(Arg1, Op2TmpLValue.getAddress()); |
19017 | 19012 |
|
19018 | | - int numElements = Op0VecTy -> getNumElements() * 2; |
| 19013 | + } else { |
19019 | 19014 |
|
19020 | | - FixedVectorType *destTy = FixedVectorType::get(Int32Ty, numElements); |
19021 | | - |
19022 | | - Value *bitcast = Builder.CreateBitCast(Op0, destTy); |
| 19015 | + assert(!CGM.getTarget().getTriple().isDXIL() && |
| 19016 | + "For non-DXIL targets we generate the instructions"); |
19023 | 19017 |
|
19024 | | - SmallVector<int> lowbitsIndex; |
19025 | | - SmallVector<int> highbitsIndex; |
| 19018 | + if (!Op0->getType()->isVectorTy()) { |
| 19019 | + FixedVectorType *DestTy = FixedVectorType::get(Int32Ty, 2); |
| 19020 | + Value *Bitcast = Builder.CreateBitCast(Op0, DestTy); |
19026 | 19021 |
|
19027 | | - for(int idx = 0; idx < numElements; idx += 2){ |
19028 | | - lowbitsIndex.push_back(idx); |
19029 | | - } |
| 19022 | + Value *Arg0 = Builder.CreateExtractElement(Bitcast, 0.0); |
| 19023 | + Value *Arg1 = Builder.CreateExtractElement(Bitcast, 1.0); |
19030 | 19024 |
|
19031 | | - for(int idx = 1; idx < numElements; idx += 2){ |
19032 | | - highbitsIndex.push_back(idx); |
19033 | | - } |
| 19025 | + Builder.CreateStore(Arg0, Op1TmpLValue.getAddress()); |
| 19026 | + LastInst = Builder.CreateStore(Arg1, Op2TmpLValue.getAddress()); |
| 19027 | + } else { |
19034 | 19028 |
|
19035 | | - Value *arg0 = Builder.CreateShuffleVector(bitcast, lowbitsIndex); |
19036 | | - Value *arg1 = Builder.CreateShuffleVector(bitcast, highbitsIndex); |
| 19029 | + const auto *TargTy = E->getArg(0)->getType()->getAs<VectorType>(); |
| 19030 | + |
| 19031 | + int NumElements = TargTy->getNumElements(); |
| 19032 | + |
| 19033 | + FixedVectorType *DestTy = FixedVectorType::get(Int32Ty, 4); |
| 19034 | + if (NumElements == 1) { |
| 19035 | + FixedVectorType *DestTy = FixedVectorType::get(Int32Ty, 2); |
| 19036 | + Value *Bitcast = Builder.CreateBitCast(Op0, DestTy); |
| 19037 | + |
| 19038 | + Value *Arg0 = Builder.CreateExtractElement(Bitcast, 0.0); |
| 19039 | + Value *Arg1 = Builder.CreateExtractElement(Bitcast, 1.0); |
| 19040 | + |
| 19041 | + Builder.CreateStore(Arg0, Op1TmpLValue.getAddress()); |
| 19042 | + LastInst = Builder.CreateStore(Arg1, Op2TmpLValue.getAddress()); |
| 19043 | + } else if (NumElements == 2) { |
| 19044 | + auto [LowBits, HighBits] = EmitVectorCode(Op0, &Builder, DestTy); |
| 19045 | + |
| 19046 | + Builder.CreateStore(LowBits, Op1TmpLValue.getAddress()); |
| 19047 | + LastInst = Builder.CreateStore(HighBits, Op2TmpLValue.getAddress()); |
| 19048 | + } else { |
| 19049 | + |
| 19050 | + SmallVector<std::pair<Value *, Value *>> EmitedValuePairs; |
| 19051 | + |
| 19052 | + for (int It = 0; It < NumElements; It += 2) { |
| 19053 | + // Due to existing restrictions to SPIR-V and splitdouble, |
| 19054 | + // all shufflevector operations, should return vectors of |
| 19055 | + // the same size, up to 4. Such introduce and edge case |
| 19056 | + // when we got odd sized vectors, which will require |
| 19057 | + // an additional dummy value, such is masked out in a later |
| 19058 | + // stage of this code. |
| 19059 | + auto Shuff = Builder.CreateShuffleVector( |
| 19060 | + Op0, {It, ((It + 1) % NumElements)}); |
| 19061 | + std::pair<Value *, Value *> ValuePair = |
| 19062 | + EmitVectorCode(Shuff, &Builder, DestTy); |
| 19063 | + EmitedValuePairs.push_back(ValuePair); |
| 19064 | + } |
19037 | 19065 |
|
19038 | | - Builder.CreateStore(arg0, Op1TmpLValue.getAddress()); |
19039 | | - auto *s = Builder.CreateStore(arg1, Op2TmpLValue.getAddress()); |
| 19066 | + SmallVector<int> Index; |
| 19067 | + auto InitSize = NumElements > 4 ? 4 : NumElements; |
| 19068 | + for (int It = 0; It < InitSize; It++) |
| 19069 | + Index.push_back(It); |
| 19070 | + |
| 19071 | + auto arg0 = Builder.CreateShuffleVector( |
| 19072 | + EmitedValuePairs[0].first, EmitedValuePairs[1].first, Index); |
| 19073 | + auto arg1 = Builder.CreateShuffleVector( |
| 19074 | + EmitedValuePairs[0].second, EmitedValuePairs[1].second, Index); |
| 19075 | + |
| 19076 | + for (size_t It = 2; It < EmitedValuePairs.size(); It++) { |
| 19077 | + int CurIndexSize = Index.size(); |
| 19078 | + Index.insert(Index.end(), {CurIndexSize + 1, |
| 19079 | + ((CurIndexSize + 2) % NumElements)}); |
| 19080 | + arg0 = Builder.CreateShuffleVector(arg0, EmitedValuePairs[It].first, |
| 19081 | + Index); |
| 19082 | + arg1 = Builder.CreateShuffleVector( |
| 19083 | + arg1, EmitedValuePairs[It].second, Index); |
| 19084 | + } |
19040 | 19085 |
|
| 19086 | + Builder.CreateStore(arg0, Op1TmpLValue.getAddress()); |
| 19087 | + LastInst = Builder.CreateStore(arg1, Op2TmpLValue.getAddress()); |
| 19088 | + } |
| 19089 | + } |
| 19090 | + } |
19041 | 19091 | EmitWritebacks(*this, Args); |
19042 | | - return s; |
19043 | | - |
| 19092 | + return LastInst; |
19044 | 19093 | } |
19045 | 19094 | } |
19046 | 19095 | return nullptr; |
|
0 commit comments