|
14 | 14 | #include "SystemZCallingConv.h" |
15 | 15 | #include "SystemZConstantPoolValue.h" |
16 | 16 | #include "SystemZMachineFunctionInfo.h" |
17 | | -#include "SystemZRegisterInfo.h" |
18 | 17 | #include "SystemZTargetMachine.h" |
19 | 18 | #include "llvm/ADT/SmallSet.h" |
20 | 19 | #include "llvm/CodeGen/CallingConvLower.h" |
21 | 20 | #include "llvm/CodeGen/ISDOpcodes.h" |
22 | 21 | #include "llvm/CodeGen/MachineInstrBuilder.h" |
23 | | -#include "llvm/CodeGen/MachineOperand.h" |
| 22 | +#include "llvm/CodeGen/MachineRegisterInfo.h" |
24 | 23 | #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" |
25 | 24 | #include "llvm/IR/GlobalAlias.h" |
26 | 25 | #include "llvm/IR/IntrinsicInst.h" |
@@ -1971,6 +1970,28 @@ SDValue SystemZTargetLowering::joinRegisterPartsIntoValue( |
1971 | 1970 | return SDValue(); |
1972 | 1971 | } |
1973 | 1972 |
|
| 1973 | +// The first part of a split stack argument is at index I in Args (and |
| 1974 | +// ArgLocs). Return the type of a part and the number of them by reference. |
| 1975 | +template <class ArgTy> |
| 1976 | +static bool analyzeArgSplit(const SmallVectorImpl<ArgTy> &Args, |
| 1977 | + SmallVector<CCValAssign, 16> &ArgLocs, unsigned I, |
| 1978 | + MVT &PartVT, unsigned &NumParts) { |
| 1979 | + if (!Args[I].Flags.isSplit()) |
| 1980 | + return false; |
| 1981 | + assert(I < ArgLocs.size() && ArgLocs.size() == Args.size() && |
| 1982 | + "ArgLocs havoc."); |
| 1983 | + PartVT = ArgLocs[I].getValVT(); |
| 1984 | + NumParts = 1; |
| 1985 | + for (unsigned PartIdx = I + 1;; ++PartIdx) { |
| 1986 | + assert(PartIdx != ArgLocs.size() && "SplitEnd not found."); |
| 1987 | + assert(ArgLocs[PartIdx].getValVT() == PartVT && "Unsupported split."); |
| 1988 | + ++NumParts; |
| 1989 | + if (Args[PartIdx].Flags.isSplitEnd()) |
| 1990 | + break; |
| 1991 | + } |
| 1992 | + return true; |
| 1993 | +} |
| 1994 | + |
1974 | 1995 | SDValue SystemZTargetLowering::LowerFormalArguments( |
1975 | 1996 | SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, |
1976 | 1997 | const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, |
@@ -2075,16 +2096,26 @@ SDValue SystemZTargetLowering::LowerFormalArguments( |
2075 | 2096 | MachinePointerInfo())); |
2076 | 2097 | // If the original argument was split (e.g. i128), we need |
2077 | 2098 | // to load all parts of it here (using the same address). |
2078 | | - unsigned ArgIndex = Ins[I].OrigArgIndex; |
2079 | | - assert (Ins[I].PartOffset == 0); |
2080 | | - while (I + 1 != E && Ins[I + 1].OrigArgIndex == ArgIndex) { |
2081 | | - CCValAssign &PartVA = ArgLocs[I + 1]; |
2082 | | - unsigned PartOffset = Ins[I + 1].PartOffset; |
2083 | | - SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue, |
2084 | | - DAG.getIntPtrConstant(PartOffset, DL)); |
2085 | | - InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address, |
2086 | | - MachinePointerInfo())); |
2087 | | - ++I; |
| 2099 | + MVT PartVT; |
| 2100 | + unsigned NumParts; |
| 2101 | + if (analyzeArgSplit(Ins, ArgLocs, I, PartVT, NumParts)) { |
| 2102 | + // TODO: It is strange that while LowerCallTo() sets the PartOffset |
| 2103 | + // relative to the first split part LowerArguments() sets the offset |
| 2104 | + // from the beginning of the struct. So with {i32, i256}, the |
| 2105 | + // PartOffset for the i256 parts are differently handled. Try to |
| 2106 | + // remove that difference and use PartOffset directly here (instead |
| 2107 | + // of SplitBaseOffs). |
| 2108 | + unsigned SplitBaseOffs = Ins[I].PartOffset; |
| 2109 | + for (unsigned PartIdx = 1; PartIdx < NumParts; ++PartIdx) { |
| 2110 | + ++I; |
| 2111 | + CCValAssign &PartVA = ArgLocs[I]; |
| 2112 | + unsigned PartOffset = Ins[I].PartOffset - SplitBaseOffs; |
| 2113 | + SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue, |
| 2114 | + DAG.getIntPtrConstant(PartOffset, DL)); |
| 2115 | + InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address, |
| 2116 | + MachinePointerInfo())); |
| 2117 | + assert(PartOffset && "Offset should be non-zero."); |
| 2118 | + } |
2088 | 2119 | } |
2089 | 2120 | } else |
2090 | 2121 | InVals.push_back(convertLocVTToValVT(DAG, DL, VA, Chain, ArgValue)); |
@@ -2320,37 +2351,33 @@ SystemZTargetLowering::LowerCall(CallLoweringInfo &CLI, |
2320 | 2351 |
|
2321 | 2352 | if (VA.getLocInfo() == CCValAssign::Indirect) { |
2322 | 2353 | // Store the argument in a stack slot and pass its address. |
2323 | | - unsigned ArgIndex = Outs[I].OrigArgIndex; |
2324 | 2354 | EVT SlotVT; |
2325 | | - if (I + 1 != E && Outs[I + 1].OrigArgIndex == ArgIndex) { |
2326 | | - // Allocate the full stack space for a promoted (and split) argument. |
2327 | | - Type *OrigArgType = CLI.Args[Outs[I].OrigArgIndex].Ty; |
2328 | | - EVT OrigArgVT = getValueType(MF.getDataLayout(), OrigArgType); |
2329 | | - MVT PartVT = getRegisterTypeForCallingConv(Ctx, CLI.CallConv, OrigArgVT); |
2330 | | - unsigned N = getNumRegistersForCallingConv(Ctx, CLI.CallConv, OrigArgVT); |
2331 | | - SlotVT = EVT::getIntegerVT(Ctx, PartVT.getSizeInBits() * N); |
2332 | | - } else { |
| 2355 | + MVT PartVT; |
| 2356 | + unsigned NumParts = 1; |
| 2357 | + if (analyzeArgSplit(Outs, ArgLocs, I, PartVT, NumParts)) |
| 2358 | + SlotVT = EVT::getIntegerVT(Ctx, PartVT.getSizeInBits() * NumParts); |
| 2359 | + else |
2333 | 2360 | SlotVT = Outs[I].VT; |
2334 | | - } |
2335 | 2361 | SDValue SpillSlot = DAG.CreateStackTemporary(SlotVT); |
2336 | 2362 | int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex(); |
2337 | 2363 | MemOpChains.push_back( |
2338 | 2364 | DAG.getStore(Chain, DL, ArgValue, SpillSlot, |
2339 | 2365 | MachinePointerInfo::getFixedStack(MF, FI))); |
2340 | 2366 | // If the original argument was split (e.g. i128), we need |
2341 | 2367 | // to store all parts of it here (and pass just one address). |
2342 | | - assert (Outs[I].PartOffset == 0); |
2343 | | - while (I + 1 != E && Outs[I + 1].OrigArgIndex == ArgIndex) { |
2344 | | - SDValue PartValue = OutVals[I + 1]; |
2345 | | - unsigned PartOffset = Outs[I + 1].PartOffset; |
| 2368 | + assert(Outs[I].PartOffset == 0); |
| 2369 | + for (unsigned PartIdx = 1; PartIdx < NumParts; ++PartIdx) { |
| 2370 | + ++I; |
| 2371 | + SDValue PartValue = OutVals[I]; |
| 2372 | + unsigned PartOffset = Outs[I].PartOffset; |
2346 | 2373 | SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot, |
2347 | 2374 | DAG.getIntPtrConstant(PartOffset, DL)); |
2348 | 2375 | MemOpChains.push_back( |
2349 | 2376 | DAG.getStore(Chain, DL, PartValue, Address, |
2350 | 2377 | MachinePointerInfo::getFixedStack(MF, FI))); |
| 2378 | + assert(PartOffset && "Offset should be non-zero."); |
2351 | 2379 | assert((PartOffset + PartValue.getValueType().getStoreSize() <= |
2352 | 2380 | SlotVT.getStoreSize()) && "Not enough space for argument part!"); |
2353 | | - ++I; |
2354 | 2381 | } |
2355 | 2382 | ArgValue = SpillSlot; |
2356 | 2383 | } else |
@@ -2535,7 +2562,7 @@ bool SystemZTargetLowering::CanLowerReturn( |
2535 | 2562 | // Special case that we cannot easily detect in RetCC_SystemZ since |
2536 | 2563 | // i128 may not be a legal type. |
2537 | 2564 | for (auto &Out : Outs) |
2538 | | - if (Out.ArgVT == MVT::i128) |
| 2565 | + if (Out.ArgVT.isScalarInteger() && Out.ArgVT.getSizeInBits() > 64) |
2539 | 2566 | return false; |
2540 | 2567 |
|
2541 | 2568 | SmallVector<CCValAssign, 16> RetLocs; |
@@ -8027,25 +8054,6 @@ SDValue SystemZTargetLowering::combineSTORE( |
8027 | 8054 | SN->getMemOperand()); |
8028 | 8055 | } |
8029 | 8056 | } |
8030 | | - |
8031 | | - // combine STORE (LOAD_STACK_GUARD) into MOVE_STACK_GUARD |
8032 | | - if (Op1->isMachineOpcode() && |
8033 | | - (Op1->getMachineOpcode() == SystemZ::LOAD_STACK_GUARD)) { |
8034 | | - // If so, create a MOVE_STACK_GUARD node to replace the store, |
8035 | | - // and a LOAD_STACK_GUARD_ADDRESS to replace the LOAD_STACK_GUARD |
8036 | | - MachineSDNode *LoadAddr = DAG.getMachineNode( |
8037 | | - SystemZ::LOAD_STACK_GUARD_ADDRESS, SDLoc(SN), MVT::i64); |
8038 | | - int FI = cast<FrameIndexSDNode>(SN->getOperand(2))->getIndex(); |
8039 | | - // FrameIndex, Dummy Displacement |
8040 | | - SDValue Ops[] = {DAG.getTargetFrameIndex(FI, MVT::i64), |
8041 | | - DAG.getTargetConstant(0, SDLoc(SN), MVT::i64), |
8042 | | - SDValue(LoadAddr, 0), SN->getChain()}; |
8043 | | - MachineSDNode *Move = DAG.getMachineNode(SystemZ::MOVE_STACK_GUARD, |
8044 | | - SDLoc(SN), MVT::Other, Ops); |
8045 | | - |
8046 | | - return SDValue(Move, 0); |
8047 | | - } |
8048 | | - |
8049 | 8057 | // Combine STORE (BSWAP) into STRVH/STRV/STRVG/VSTBR |
8050 | 8058 | if (!SN->isTruncatingStore() && |
8051 | 8059 | Op1.getOpcode() == ISD::BSWAP && |
@@ -8865,103 +8873,25 @@ SystemZTargetLowering::getJumpConditionMergingParams(Instruction::BinaryOps Opc, |
8865 | 8873 | return {-1, -1, -1}; |
8866 | 8874 | } |
8867 | 8875 |
|
8868 | | -namespace { |
8869 | | -bool isStackGuardCheck(SDNode const *N, int &FI, SDValue &InChain, |
8870 | | - SDValue &OutChain, SDValue &StackGuardLoad, |
8871 | | - SystemZTargetLowering::DAGCombinerInfo &DCI) { |
8872 | | - auto Comp = N->getOperand(4); |
8873 | | - if (Comp->getOpcode() != SystemZISD::ICMP) |
8874 | | - return false; |
8875 | | - |
8876 | | - if (!Comp->hasOneUse()) |
8877 | | - return false; |
8878 | | - |
8879 | | - SDValue LHS = Comp->getOperand(0); |
8880 | | - SDValue RHS = Comp->getOperand(1); |
8881 | | - LoadSDNode *FILoad; |
8882 | | - |
8883 | | - if (LHS.isMachineOpcode() && |
8884 | | - LHS.getMachineOpcode() == SystemZ::LOAD_STACK_GUARD && |
8885 | | - ISD::isNormalLoad(RHS.getNode()) && |
8886 | | - dyn_cast<FrameIndexSDNode>(RHS.getOperand(1))) { |
8887 | | - StackGuardLoad = LHS; |
8888 | | - FILoad = cast<LoadSDNode>(RHS); |
8889 | | - } else if ((RHS.isMachineOpcode() && |
8890 | | - RHS.getMachineOpcode() == SystemZ::LOAD_STACK_GUARD && |
8891 | | - ISD::isNormalLoad(LHS.getNode()) && |
8892 | | - dyn_cast<FrameIndexSDNode>(LHS.getOperand(1)))) { |
8893 | | - StackGuardLoad = RHS; |
8894 | | - FILoad = cast<LoadSDNode>(LHS); |
8895 | | - } else |
8896 | | - return false; |
8897 | | - |
8898 | | - // Assert that the values of the loads are not used elsewhere. |
8899 | | - // Bail for now. TODO: What is the proper response here? |
8900 | | - assert( |
8901 | | - SDValue(FILoad, 0).hasOneUse() && |
8902 | | - "Value of stackguard loaded from stack must be used for compare only!"); |
8903 | | - assert(StackGuardLoad.hasOneUse() && |
8904 | | - "Value of reference stackguard must be used for compare only!"); |
8905 | | - |
8906 | | - FI = cast<FrameIndexSDNode>(FILoad->getOperand(1))->getIndex(); |
8907 | | - InChain = FILoad->getChain(); |
8908 | | - OutChain = SDValue(FILoad, 1); |
8909 | | - DCI.AddToWorklist(FILoad); |
8910 | | - DCI.AddToWorklist(Comp.getNode()); |
8911 | | - return true; |
8912 | | -} |
8913 | | -} // namespace |
8914 | | - |
8915 | 8876 | SDValue SystemZTargetLowering::combineBR_CCMASK(SDNode *N, |
8916 | 8877 | DAGCombinerInfo &DCI) const { |
8917 | 8878 | SelectionDAG &DAG = DCI.DAG; |
8918 | 8879 |
|
| 8880 | + // Combine BR_CCMASK (ICMP (SELECT_CCMASK)) into a single BR_CCMASK. |
8919 | 8881 | auto *CCValid = dyn_cast<ConstantSDNode>(N->getOperand(1)); |
8920 | 8882 | auto *CCMask = dyn_cast<ConstantSDNode>(N->getOperand(2)); |
8921 | 8883 | if (!CCValid || !CCMask) |
8922 | 8884 | return SDValue(); |
| 8885 | + |
8923 | 8886 | int CCValidVal = CCValid->getZExtValue(); |
8924 | 8887 | int CCMaskVal = CCMask->getZExtValue(); |
8925 | 8888 | SDValue Chain = N->getOperand(0); |
8926 | 8889 | SDValue CCReg = N->getOperand(4); |
8927 | | - SDLoc DL(N); |
8928 | | - |
8929 | | - // Combine BR_CCMASK (ICMP (Load FI, Load StackGuard)) into BRC |
8930 | | - // (COMPARE_STACK_GUARD) |
8931 | | - int FI = 0; |
8932 | | - SDValue InChain, OutChain, StackGuardLoad; |
8933 | | - if (isStackGuardCheck(N, FI, InChain, OutChain, StackGuardLoad, DCI)) { |
8934 | | - // Sanity Checks |
8935 | | - assert(CCMaskVal == SystemZ::CCMASK_CMP_NE && |
8936 | | - "Unexpected branch condition in stack guard check"); |
8937 | | - // Handle the load's chain if necessary |
8938 | | - DAG.ReplaceAllUsesOfValueWith(OutChain, InChain); |
8939 | | - |
8940 | | - // Construct the LOAD_STACK_GUARD_ADDRESS node to replace LOAD_STACK_GUARD |
8941 | | - auto *LoadAddress = |
8942 | | - DAG.getMachineNode(SystemZ::LOAD_STACK_GUARD_ADDRESS, DL, MVT::i64); |
8943 | | - |
8944 | | - // Construct the COMPARE_STACK_GUARD node |
8945 | | - SDVTList CmpVTs = DAG.getVTList(MVT::Other, MVT::Glue); |
8946 | | - auto CompOps = {DAG.getTargetFrameIndex(FI, MVT::i64), |
8947 | | - DAG.getTargetConstant(0, DL, MVT::i64), |
8948 | | - SDValue(LoadAddress, 0), InChain}; |
8949 | | - auto *Compare = |
8950 | | - DAG.getMachineNode(SystemZ::COMPARE_STACK_GUARD, DL, CmpVTs, CompOps); |
8951 | | - // Construct the BRC node using COMPARE_STACK_GUARD's CC result |
8952 | | - auto BranchOps = {DAG.getTargetConstant(CCValidVal, DL, MVT::i32), |
8953 | | - DAG.getTargetConstant(CCMaskVal, DL, MVT::i32), |
8954 | | - N->getOperand(3), SDValue(Compare, 0), |
8955 | | - SDValue(Compare, 1)}; |
8956 | | - return SDValue(DAG.getMachineNode(SystemZ::BRC, DL, MVT::Other, BranchOps), |
8957 | | - 0); |
8958 | | - } |
8959 | | - |
8960 | | - // Combine BR_CCMASK (ICMP (SELECT_CCMASK)) into a single BR_CCMASK. |
8961 | 8890 | if (combineCCMask(CCReg, CCValidVal, CCMaskVal, DAG)) |
8962 | | - return DAG.getNode(SystemZISD::BR_CCMASK, DL, N->getValueType(0), Chain, |
8963 | | - DAG.getTargetConstant(CCValidVal, DL, MVT::i32), |
8964 | | - DAG.getTargetConstant(CCMaskVal, DL, MVT::i32), |
| 8891 | + return DAG.getNode(SystemZISD::BR_CCMASK, SDLoc(N), N->getValueType(0), |
| 8892 | + Chain, |
| 8893 | + DAG.getTargetConstant(CCValidVal, SDLoc(N), MVT::i32), |
| 8894 | + DAG.getTargetConstant(CCMaskVal, SDLoc(N), MVT::i32), |
8965 | 8895 | N->getOperand(3), CCReg); |
8966 | 8896 | return SDValue(); |
8967 | 8897 | } |
@@ -9368,8 +9298,6 @@ SDValue SystemZTargetLowering::PerformDAGCombine(SDNode *N, |
9368 | 9298 | case SystemZISD::BR_CCMASK: return combineBR_CCMASK(N, DCI); |
9369 | 9299 | case SystemZISD::SELECT_CCMASK: return combineSELECT_CCMASK(N, DCI); |
9370 | 9300 | case SystemZISD::GET_CCMASK: return combineGET_CCMASK(N, DCI); |
9371 | | - // case SystemZISD::ICMP: |
9372 | | - // return combineICMP(N, DCI); |
9373 | 9301 | case ISD::SRL: |
9374 | 9302 | case ISD::SRA: return combineShiftToMulAddHigh(N, DCI); |
9375 | 9303 | case ISD::MUL: return combineMUL(N, DCI); |
|
0 commit comments