[PowerPC] Initial support of tail call optimization on AIX tail call

llvm/lib/Target/PowerPC/PPCFrameLowering.cpp

@@ -1935,6 +1935,9 @@ void PPCFrameLowering::createTailCallBranchInstr(MachineBasicBlock &MBB) const {
     else if (JumpTarget.isSymbol())
       BuildMI(MBB, MBBI, dl, TII.get(PPC::TAILB)).
         addExternalSymbol(JumpTarget.getSymbolName());
+    else if (JumpTarget.isMCSymbol())
+      BuildMI(MBB, MBBI, dl, TII.get(PPC::TAILB))
+          .addSym(JumpTarget.getMCSymbol());
     else
       llvm_unreachable("Expecting Global or External Symbol");
   } else if (RetOpcode == PPC::TCRETURNri) {
@@ -1954,6 +1957,9 @@ void PPCFrameLowering::createTailCallBranchInstr(MachineBasicBlock &MBB) const {
     else if (JumpTarget.isSymbol())
       BuildMI(MBB, MBBI, dl, TII.get(PPC::TAILB8)).
         addExternalSymbol(JumpTarget.getSymbolName());
+    else if (JumpTarget.isMCSymbol())
+      BuildMI(MBB, MBBI, dl, TII.get(PPC::TAILB8))
+          .addSym(JumpTarget.getMCSymbol());
     else
       llvm_unreachable("Expecting Global or External Symbol");
   } else if (RetOpcode == PPC::TCRETURNri8) {

llvm/lib/Target/PowerPC/PPCISelLowering.cpp

@@ -5082,9 +5082,8 @@ static bool hasSameArgumentList(const Function *CallerFn, const CallBase &CB) {
 
 // Returns true if TCO is possible between the callers and callees
 // calling conventions.
-static bool
-areCallingConvEligibleForTCO_64SVR4(CallingConv::ID CallerCC,
-                                    CallingConv::ID CalleeCC) {
+static bool areCallingConvEligibleForTCO(CallingConv::ID CallerCC,
+                                         CallingConv::ID CalleeCC) {
   // Tail calls are possible with fastcc and ccc.
   auto isTailCallableCC  = [] (CallingConv::ID CC){
       return  CC == CallingConv::C || CC == CallingConv::Fast;
@@ -5113,7 +5112,7 @@ bool PPCTargetLowering::IsEligibleForTailCallOptimization_64SVR4(
   if (isVarArg) return false;
 
   // Check that the calling conventions are compatible for tco.
-  if (!areCallingConvEligibleForTCO_64SVR4(CallerCC, CalleeCC))
+  if (!areCallingConvEligibleForTCO(CallerCC, CalleeCC))
     return false;
 
   // Caller contains any byval parameter is not supported.
@@ -5183,14 +5182,124 @@ bool PPCTargetLowering::IsEligibleForTailCallOptimization_64SVR4(
   return true;
 }
 
+static bool
+needStackSlotPassParameters_AIX(const PPCSubtarget &Subtarget,
+                                const SmallVectorImpl<ISD::OutputArg> &Outs) {
+  const bool IsPPC64 = Subtarget.isPPC64();
+  const Align PtrAlign = IsPPC64 ? Align(8) : Align(4);
+  const unsigned PhyGPRsNum = 8;
+  const unsigned PhyVRsNum = 12;
+  unsigned PhyGPRAllocated = 0;
+  unsigned PhyVRAllocated = 0;
+
+  for (unsigned i = 0; i != Outs.size(); ++i) {
+    MVT ArgVT = Outs[i].VT;
+    ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
+    if (ArgFlags.isByVal()) {
+      const unsigned ByValSize = ArgFlags.getByValSize();
+      const unsigned StackSize = alignTo(ByValSize, PtrAlign);
+      PhyGPRAllocated += StackSize / PtrAlign.value();
+      if (PhyGPRAllocated > PhyGPRsNum)
+        return true;
+      continue;
+    }
+
+    switch (ArgVT.SimpleTy) {
+    default:
+      report_fatal_error("Unhandled value type for argument.");
+    case MVT::i64:
+      // i64 arguments should have been split to i32 for PPC32.
+      assert(IsPPC64 && "PPC32 should have split i64 values.");
+      [[fallthrough]];
+    case MVT::i1:
+    case MVT::i32:
+      if (++PhyGPRAllocated > PhyGPRsNum)
+        return true;
+      break;
+    case MVT::f32:
+    case MVT::f64: {
+      const unsigned StoreSize = ArgVT.getStoreSize();
+      PhyGPRAllocated += StoreSize / PtrAlign.value();
+      if (PhyGPRAllocated > PhyGPRsNum)
+        return true;
+      break;
+    }
+    case MVT::v4f32:
+    case MVT::v4i32:
+    case MVT::v8i16:
+    case MVT::v16i8:
+    case MVT::v2i64:
+    case MVT::v2f64:
+    case MVT::v1i128:
+      if (++PhyVRAllocated > PhyVRsNum)
+        return true;
+    }
+  }
+
+  return false;
+}
+
+bool PPCTargetLowering::IsEligibleForTailCallOptimization_AIX(
+    const GlobalValue *CalleeGV, CallingConv::ID CalleeCC,
+    CallingConv::ID CallerCC, const CallBase *CB, bool isVarArg,
+    const SmallVectorImpl<ISD::OutputArg> &Outs, const Function *CallerFunc,
+    bool isCalleeExternalSymbol) const {
+  bool TailCallOpt = getTargetMachine().Options.GuaranteedTailCallOpt;
+
+  if (DisableSCO && !TailCallOpt)
+    return false;
+
+  // Variadic argument functions are not supported.
+  if (isVarArg)
+    return false;
+
+  // Check that the calling conventions are compatible for tco.
+  if (!areCallingConvEligibleForTCO(CallerCC, CalleeCC))
+    return false;
+
+  if (!Subtarget.isUsingPCRelativeCalls() &&
+      !isFunctionGlobalAddress(CalleeGV) && !isCalleeExternalSymbol)
+    return false;
+
+  // TCO allows altering callee ABI, so we don't have to check further.
+  if (CalleeCC == CallingConv::Fast && TailCallOpt)
+    return true;
+
+  if (DisableSCO)
+    return false;
+
+  if (CallerCC != CalleeCC && needStackSlotPassParameters_AIX(Subtarget, Outs))
+    return false;
+
+  // If callee use the same argument list that caller is using, then we can
+  // apply SCO on this case. If it is not, then we need to check if callee needs
+  // stack for passing arguments.
+  // PC Relative tail calls may not have a CallBase.
+  // If there is no CallBase we cannot verify if we have the same argument
+  // list so assume that we don't have the same argument list.
+  if (CB && !hasSameArgumentList(CallerFunc, *CB) &&
+      needStackSlotPassParameters_AIX(Subtarget, Outs))
+    return false;
+  else if (!CB && needStackSlotPassParameters_AIX(Subtarget, Outs))
+    return false;
+
+  return true;
+}
+
 /// IsEligibleForTailCallOptimization - Check whether the call is eligible
 /// for tail call optimization. Targets which want to do tail call
 /// optimization should implement this function.
 bool PPCTargetLowering::IsEligibleForTailCallOptimization(
     const GlobalValue *CalleeGV, CallingConv::ID CalleeCC,
     CallingConv::ID CallerCC, bool isVarArg,
     const SmallVectorImpl<ISD::InputArg> &Ins) const {
-  if (!getTargetMachine().Options.GuaranteedTailCallOpt)
+  bool TailCallOpt = getTargetMachine().Options.GuaranteedTailCallOpt;
+
+  // Enable SCO on AIX.
+  if (!TailCallOpt && !Subtarget.isAIXABI())
+    return false;
+
+  if (DisableSCO)
     return false;
 
   // Variable argument functions are not supported.
@@ -5869,6 +5978,7 @@ SDValue PPCTargetLowering::FinishCall(
             Callee.getOpcode() == ISD::TargetExternalSymbol ||
             Callee.getOpcode() == ISD::TargetGlobalAddress ||
             isa<ConstantSDNode>(Callee) ||
+            (Subtarget.isAIXABI() && Callee.getOpcode() == ISD::MCSymbol) ||
             (CFlags.IsIndirect && Subtarget.isUsingPCRelativeCalls())) &&
            "Expecting a global address, external symbol, absolute value, "
            "register or an indirect tail call when PC Relative calls are "
@@ -5936,9 +6046,14 @@ bool PPCTargetLowering::isEligibleForTCO(
     return IsEligibleForTailCallOptimization_64SVR4(
         CalleeGV, CalleeCC, CallerCC, CB, isVarArg, Outs, Ins, CallerFunc,
         isCalleeExternalSymbol);
-  else
-    return IsEligibleForTailCallOptimization(CalleeGV, CalleeCC, CallerCC,
-                                             isVarArg, Ins);
+
+  else if (Subtarget.isAIXABI())
+    return IsEligibleForTailCallOptimization_AIX(CalleeGV, CalleeCC, CallerCC,
+                                                 CB, isVarArg, Outs, CallerFunc,
+                                                 isCalleeExternalSymbol);
+
+  return IsEligibleForTailCallOptimization(CalleeGV, CalleeCC, CallerCC,
+                                           isVarArg, Ins);
 }
 
 SDValue
@@ -7244,9 +7359,6 @@ SDValue PPCTargetLowering::LowerFormalArguments_AIX(
           CallConv == CallingConv::Fast) &&
          "Unexpected calling convention!");
 
-  if (getTargetMachine().Options.GuaranteedTailCallOpt)
-    report_fatal_error("Tail call support is unimplemented on AIX.");
-
   if (useSoftFloat())
     report_fatal_error("Soft float support is unimplemented on AIX.");
 
@@ -7616,6 +7728,8 @@ SDValue PPCTargetLowering::LowerCall_AIX(
   // The LSA is 24 bytes (6x4) in PPC32 and 48 bytes (6x8) in PPC64.
   const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
   const bool IsPPC64 = Subtarget.isPPC64();
+  bool IsSibCall =
+      CFlags.IsTailCall && !getTargetMachine().Options.GuaranteedTailCallOpt;
   const EVT PtrVT = getPointerTy(DAG.getDataLayout());
   const unsigned PtrByteSize = IsPPC64 ? 8 : 4;
   CCInfo.AllocateStack(LinkageSize, Align(PtrByteSize));
@@ -7631,9 +7745,15 @@ SDValue PPCTargetLowering::LowerCall_AIX(
   const unsigned NumBytes = std::max<unsigned>(
       LinkageSize + MinParameterSaveAreaSize, CCInfo.getStackSize());
 
+  // To protect arguments on the stack from being clobbered in a tail call,
+  // force all the loads to happen before doing any other lowering.
+  if (CFlags.IsTailCall)
+    Chain = DAG.getStackArgumentTokenFactor(Chain);
+
   // Adjust the stack pointer for the new arguments...
   // These operations are automatically eliminated by the prolog/epilog pass.
-  Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl);
+  if (!IsSibCall)
+    Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl);
   SDValue CallSeqStart = Chain;
 
   SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
@@ -7821,7 +7941,6 @@ SDValue PPCTargetLowering::LowerCall_AIX(
           DAG.getStore(Chain, dl, Arg, PtrOff,
                        MachinePointerInfo::getStack(MF, VA.getLocMemOffset()),
                        Subtarget.getFrameLowering()->getStackAlign()));
-
       continue;
     }
 
@@ -7903,6 +8022,12 @@ SDValue PPCTargetLowering::LowerCall_AIX(
     InGlue = Chain.getValue(1);
   }
 
+  if (CFlags.IsTailCall && !IsSibCall) {
+    // Emit callseq_end just before tailcall node.
+  Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, InGlue, dl);
+  InGlue = Chain.getValue(1);
+  }
+
   const int SPDiff = 0;
   return FinishCall(CFlags, dl, DAG, RegsToPass, InGlue, Chain, CallSeqStart,
                     Callee, SPDiff, NumBytes, Ins, InVals, CB);
@@ -19242,9 +19367,9 @@ bool PPCTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
 
   // Make sure the callee and caller calling conventions are eligible for tco.
   const Function *Caller = CI->getParent()->getParent();
-  if (!areCallingConvEligibleForTCO_64SVR4(Caller->getCallingConv(),
-                                           CI->getCallingConv()))
-      return false;
+  if (!areCallingConvEligibleForTCO(Caller->getCallingConv(),
+                                    CI->getCallingConv()))
+    return false;
 
   // If the function is local then we have a good chance at tail-calling it
   return getTargetMachine().shouldAssumeDSOLocal(Callee);

llvm/lib/Target/PowerPC/PPCISelLowering.h

@@ -1250,6 +1250,12 @@ namespace llvm {
         CallingConv::ID CallerCC, bool isVarArg,
         const SmallVectorImpl<ISD::InputArg> &Ins) const;
 
+    bool IsEligibleForTailCallOptimization_AIX(
+        const GlobalValue *CalleeGV, CallingConv::ID CalleeCC,
+        CallingConv::ID CallerCC, const CallBase *CB, bool isVarArg,
+        const SmallVectorImpl<ISD::OutputArg> &Outs, const Function *CallerFunc,
+        bool isCalleeExternalSymbol) const;
+
     bool IsEligibleForTailCallOptimization_64SVR4(
         const GlobalValue *CalleeGV, CallingConv::ID CalleeCC,
         CallingConv::ID CallerCC, const CallBase *CB, bool isVarArg,

llvm/lib/Target/PowerPC/PPCInstr64Bit.td

@@ -474,6 +474,8 @@ def : Pat<(PPCtc_return (i64 texternalsym:$dst), imm:$imm),
 def : Pat<(PPCtc_return CTRRC8:$dst, imm:$imm),
           (TCRETURNri8 CTRRC8:$dst, imm:$imm)>;
 
+def : Pat<(PPCtc_return (i64 mcsym:$dst), imm:$imm),
+          (TCRETURNdi8 mcsym:$dst, imm:$imm)>;
 
 // 64-bit CR instructions
 let Interpretation64Bit = 1, isCodeGenOnly = 1 in {

llvm/lib/Target/PowerPC/PPCInstrInfo.td

@@ -3155,6 +3155,10 @@ def : Pat<(PPCtc_return CTRRC:$dst, imm:$imm),
           (TCRETURNri CTRRC:$dst, imm:$imm)>;
 
 def : Pat<(int_ppc_fence), (FENCE)>;
+
+def : Pat<(PPCtc_return (i32 mcsym:$dst), imm:$imm),
+          (TCRETURNdi mcsym:$dst, imm:$imm)>;
+
 def : Pat<(int_ppc_readflm), (MFFS)>;
 def : Pat<(int_ppc_mffsl), (MFFSL)>;