[AIEX] Global Combiner - Pointer Modifier

llvm/lib/Target/AIE/AIE.h

@@ -54,6 +54,7 @@ FunctionPass *createAIEBaseHardwareLoopsPass();
 FunctionPass *createAIEPseudoBranchExpansion();
 FunctionPass *createAIESubRegConstrainer();
 MachineFunctionPass *createAIEClusterBaseAddress();
+MachineFunctionPass *createAIEPtrModOptimizer();
 MachineFunctionPass *createAIEAddressSpaceFlattening();
 MachineFunctionPass *createAIEEliminateDuplicatePHI();
 FunctionPass *createAIEOutlineMemoryGEP();
@@ -65,6 +66,7 @@ createDeadMachineInstructionElim(bool KeepLifetimeInstructions);
 
 void initializeAIEBaseHardwareLoopsPass(PassRegistry &);
 void initializeAIEClusterBaseAddressPass(PassRegistry &);
+void initializeAIEPtrModOptimizerPass(PassRegistry &);
 void initializeAIEAddressSpaceFlatteningPass(PassRegistry &);
 void initializeAIEEliminateDuplicatePHIPass(PassRegistry &);
 extern char &AIEFormatSelectorID;

llvm/lib/Target/AIE/AIE2InstrInfo.cpp

@@ -1503,6 +1503,24 @@ bool AIE2InstrInfo::isOffsetInImmediateRange(
   }
 }
 
+namespace {
+static const std::map<unsigned, std::set<unsigned>> S20Consumers = {
+    {Intrinsic::aie2_add_2d, {4, 5, 6, 7}},
+    {Intrinsic::aie2_add_3d, {5, 6, 7, 8, 9, 10, 11}}};
+
+static const std::map<unsigned, std::pair<unsigned, unsigned>>
+    PtrInputAndOutputIdx = {{Intrinsic::aie2_add_2d, {3, 0}},
+                            {Intrinsic::aie2_add_3d, {4, 0}}};
+
+static const AIEBaseInstrInfo::PTRModSupport AIE2PTRModSupport{
+    &S20Consumers, &PtrInputAndOutputIdx};
+
+} // namespace
+
+const AIEBaseInstrInfo::PTRModSupport &AIE2InstrInfo::getPTRModSupport() const {
+  return AIE2PTRModSupport;
+}
+
 unsigned AIE2InstrInfo::getPseudoJNZDOpcode() const { return AIE2::PseudoJNZD; }
 
 unsigned AIE2InstrInfo::getNumBypassedCycles(const InstrItineraryData *ItinData,

llvm/lib/Target/AIE/AIE2InstrInfo.h

@@ -109,6 +109,8 @@ class AIE2InstrInfo : public AIE2GenInstrInfo {
   isOffsetInImmediateRange(unsigned Opcode, unsigned LoadStoreSize,
                            std::optional<APInt> Immediate) const override;
 
+  const PTRModSupport &getPTRModSupport() const override;
+
   virtual unsigned getPseudoJNZDOpcode() const override;
 
   unsigned getNumBypassedCycles(const InstrItineraryData *ItinData,

llvm/lib/Target/AIE/AIE2PostLegalizerCustomCombiner.cpp

@@ -15,6 +15,7 @@
 
 #include "AIE2TargetMachine.h"
 #include "AIECombinerHelper.h"
+#include "AIEPtrModOptimizer.h"
 #include "llvm/CodeGen/GlobalISel/CSEInfo.h"
 #include "llvm/CodeGen/GlobalISel/Combiner.h"
 #include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
@@ -32,6 +33,8 @@
 
 using namespace llvm;
 
+extern cl::opt<bool> EnableGlobalPtrModOptimizer;
+
 static const char AIE2_POSTLEGALIZER_CUSTOM_COMBINER[] =
     "AIE2 Post Legalizer Custom Combiner";
 
@@ -45,13 +48,16 @@ class AIE2PostLegalizerCustomCombinerImpl : public Combiner {
 protected:
   // TODO: Make CombinerHelper methods const.
   mutable CombinerHelper Helper;
+  AIE::FoundCombiners EmptyGlobalCombiner;
+  AIE::FoundCombiners *GlobalCombiners = nullptr;
   const AIE2PostLegalizerCustomCombinerImplRuleConfig &RuleConfig;
   const AIE2Subtarget &STI;
 
 public:
   AIE2PostLegalizerCustomCombinerImpl(
       MachineFunction &MF, CombinerInfo &CInfo, const TargetPassConfig *TPC,
       GISelKnownBits &KB, GISelCSEInfo *CSEInfo,
+      AIE::FoundCombiners *GlobalCombiner,
       const AIE2PostLegalizerCustomCombinerImplRuleConfig &RuleConfig,
       const AIE2Subtarget &STI, MachineDominatorTree *MDT,
       const LegalizerInfo *LI);
@@ -73,17 +79,19 @@ class AIE2PostLegalizerCustomCombinerImpl : public Combiner {
 AIE2PostLegalizerCustomCombinerImpl::AIE2PostLegalizerCustomCombinerImpl(
     MachineFunction &MF, CombinerInfo &CInfo, const TargetPassConfig *TPC,
     GISelKnownBits &KB, GISelCSEInfo *CSEInfo,
+    AIE::FoundCombiners *GlobalCombiner,
     const AIE2PostLegalizerCustomCombinerImplRuleConfig &RuleConfig,
-    const AIE2Subtarget &STI,
-    MachineDominatorTree *MDT,
+    const AIE2Subtarget &STI, MachineDominatorTree *MDT,
     const LegalizerInfo *LI)
     : Combiner(MF, CInfo, TPC, &KB, CSEInfo),
       Helper(Observer, B, /*IsPostLegalize*/ false, &KB, MDT, LI),
-      RuleConfig(RuleConfig), STI(STI),
+      GlobalCombiners(GlobalCombiner), RuleConfig(RuleConfig), STI(STI),
 #define GET_GICOMBINER_CONSTRUCTOR_INITS
 #include "AIE2GenPostLegalizerGICustomCombiner.inc"
 #undef GET_GICOMBINER_CONSTRUCTOR_INITS
 {
+  if (!GlobalCombiner)
+    GlobalCombiners = &EmptyGlobalCombiner;
 }
 
 class AIE2PostLegalizerCustomCombiner : public MachineFunctionPass {
@@ -107,6 +115,9 @@ class AIE2PostLegalizerCustomCombiner : public MachineFunctionPass {
     AU.addPreserved<MachineDominatorTree>();
     AU.addRequired<GISelCSEAnalysisWrapperPass>();
     AU.addPreserved<GISelCSEAnalysisWrapperPass>();
+    if (EnableGlobalPtrModOptimizer) {
+      AU.addRequired<AIEPtrModOptimizer>();
+    }
     MachineFunctionPass::getAnalysisUsage(AU);
   }
 
@@ -145,11 +156,16 @@ bool AIE2PostLegalizerCustomCombiner::runOnMachineFunction(
   GISelKnownBits *KB = &getAnalysis<GISelKnownBitsAnalysis>().get(MF);
   MachineDominatorTree *MDT = &getAnalysis<MachineDominatorTree>();
 
+  AIE::FoundCombiners *AIEGlobalPtrIncResults = nullptr;
+  if (auto *PtrModOptPass = getAnalysisIfAvailable<AIEPtrModOptimizer>())
+    AIEGlobalPtrIncResults = PtrModOptPass->getGlobalPtrCombiners();
+
   CombinerInfo CInfo(/*AllowIllegalOps*/ true, /*ShouldLegalizeIllegal*/ false,
                      /*LegalizerInfo*/ nullptr, EnableOpt, F.hasOptSize(),
                      F.hasMinSize());
   AIE2PostLegalizerCustomCombinerImpl Impl(MF, CInfo, TPC, *KB, CSEInfo,
-                                        RuleConfig, ST, MDT, LI);
+                                           AIEGlobalPtrIncResults, RuleConfig,
+                                           ST, MDT, LI);
   return Impl.combineMachineInstrs();
 }
 

llvm/lib/Target/AIE/AIE2TargetMachine.cpp

@@ -41,6 +41,7 @@ static cl::opt<bool> EnableOutlineMemoryGEP(
     cl::desc("Enable Outlining GEPs in Memory Instructions."));
 
 extern cl::opt<bool> EnableAddressChaining;
+extern cl::opt<bool> EnableGlobalPtrModOptimizer;
 extern cl::opt<bool> EnableStagedRA;
 extern cl::opt<bool> EnableSuperRegSplitting;
 extern cl::opt<bool> AllocateMRegsFirst;
@@ -89,6 +90,8 @@ void AIE2PassConfig::addPreRegBankSelect() {
     addPass(createAIE2PostLegalizerGenericCombiner());
     if (EnableAddressChaining)
       addPass(createAIEClusterBaseAddress());
+    if (EnableGlobalPtrModOptimizer)
+      addPass(createAIEPtrModOptimizer());
     addPass(createAIE2PostLegalizerCustomCombiner());
   }
 }

llvm/lib/Target/AIE/AIEBaseInstrInfo.cpp

@@ -23,6 +23,7 @@
 #include "Utils/AIELoopUtils.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
 #include "llvm/CodeGen/LivePhysRegs.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
@@ -1288,3 +1289,89 @@ AIEBaseInstrInfo::getAlignmentBoundaries(MachineBasicBlock &MBB) const {
 
   return AlgnCandidates;
 }
+
+bool llvm::AIEBaseInstrInfo::PTRModSupport::isNativeS20Consumer(
+    const MachineInstr &MI) const {
+  switch (MI.getOpcode()) {
+  case TargetOpcode::G_PTR_ADD:
+    return true;
+  case TargetOpcode::G_INTRINSIC:
+  case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: {
+    const unsigned IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID();
+    return S20Consumers->count(IntrinsicID);
+  }
+
+  default:
+    return false;
+  }
+}
+
+bool llvm::AIEBaseInstrInfo::PTRModSupport::isNativeS20Operand(
+    const MachineInstr &MI, unsigned OperandIdx) const {
+  switch (MI.getOpcode()) {
+  case TargetOpcode::G_PTR_ADD:
+    return true;
+  case TargetOpcode::G_INTRINSIC:
+  case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: {
+    const unsigned IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID();
+    return isNativeS20ConsumerIntrinsicOperand(IntrinsicID, OperandIdx);
+  }
+  default:
+    return false;
+  }
+}
+
+bool llvm::AIEBaseInstrInfo::PTRModSupport::isNativeS20ConsumerIntrinsicOperand(
+    const unsigned IntrinsicID, unsigned OperandIdx) const {
+  auto It = S20Consumers->find(IntrinsicID);
+  if (It == S20Consumers->end())
+    return false;
+
+  const std::set<unsigned> &Indices = It->second;
+  return Indices.find(OperandIdx) != Indices.end();
+}
+
+std::optional<unsigned> llvm::AIEBaseInstrInfo::PTRModSupport::getInputPtrIdx(
+    const MachineInstr &MI) const {
+  switch (MI.getOpcode()) {
+  case TargetOpcode::G_PTR_ADD:
+    return 1;
+  case TargetOpcode::G_INTRINSIC:
+  case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: {
+    const unsigned IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID();
+    return getInputPtrIdx(IntrinsicID);
+  }
+  default:
+    return {};
+  }
+}
+
+unsigned llvm::AIEBaseInstrInfo::PTRModSupport::getInputPtrIdx(
+    const unsigned OpCode) const {
+  auto It = PtrInputAndOutputIdx->find(OpCode);
+  assert(It != PtrInputAndOutputIdx->end());
+  return It->second.first;
+}
+
+std::optional<unsigned> llvm::AIEBaseInstrInfo::PTRModSupport::getOutputPtrIdx(
+    const MachineInstr &MI) const {
+  switch (MI.getOpcode()) {
+  case TargetOpcode::G_PTR_ADD:
+    return 0;
+  case TargetOpcode::G_INTRINSIC:
+  case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: {
+    const unsigned IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID();
+    return getOutputPtrIdx(IntrinsicID);
+  }
+
+  default:
+    return {};
+  }
+}
+
+unsigned llvm::AIEBaseInstrInfo::PTRModSupport::getOutputPtrIdx(
+    const unsigned OpCode) const {
+  auto It = PtrInputAndOutputIdx->find(OpCode);
+  assert(It != PtrInputAndOutputIdx->end());
+  return It->second.second;
+}

llvm/lib/Target/AIE/AIEBaseInstrInfo.h

@@ -27,6 +27,7 @@
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <optional>
+#include <set>
 
 namespace llvm {
 
@@ -64,6 +65,43 @@ struct AIEBaseInstrInfo : public TargetInstrInfo {
     llvm_unreachable("Target didn't implement OffsetFitImmRange");
   }
 
+  class PTRModSupport {
+    /// Map between GIntrinsic and the OperandIndices that consume S20 Operands
+    const std::map<unsigned, std::set<unsigned>> *S20Consumers = nullptr;
+    /// Map between GIntrinsic/Opcode and the <Input, Output> OperandIndices
+    const std::map<unsigned, std::pair<unsigned, unsigned>>
+        *PtrInputAndOutputIdx = nullptr;
+
+    /// \return whether \p OperandIdx of \p IntrinsicId is a native S20 Operand
+    bool isNativeS20ConsumerIntrinsicOperand(const unsigned IntrinsicID,
+                                             const unsigned OperandIdx) const;
+
+    unsigned getInputPtrIdx(const unsigned OpCode) const;
+    unsigned getOutputPtrIdx(const unsigned OpCode) const;
+
+  public:
+    PTRModSupport(const std::map<unsigned, std::set<unsigned>> *S20Consumers,
+                  const std::map<unsigned, std::pair<unsigned, unsigned>>
+                      *PtrInputAndOutputIdx)
+        : S20Consumers(S20Consumers),
+          PtrInputAndOutputIdx(PtrInputAndOutputIdx) {}
+
+    /// \return whether \p MI consumes S20
+    bool isNativeS20Consumer(const MachineInstr &MI) const;
+
+    /// \return whether \p OperandIdx of \p MI is a native S20 operand
+    bool isNativeS20Operand(const MachineInstr &MI, unsigned OperandIdx) const;
+
+    std::optional<unsigned> getInputPtrIdx(const MachineInstr &MI) const;
+
+    std::optional<unsigned> getOutputPtrIdx(const MachineInstr &MI) const;
+  };
+
+  /// Return PointerModifierSupport Class for querying
+  virtual const PTRModSupport &getPTRModSupport() const {
+    llvm_unreachable("Target didn't implement getPTRModSupport");
+  }
+
   /// Return the opcode for a return instruction
   virtual unsigned getReturnOpcode() const {
     llvm_unreachable("Target didn't implement getReturnOpcode");

llvm/lib/Target/AIE/AIEBaseTargetMachine.cpp

@@ -73,6 +73,10 @@ cl::opt<bool> EnableAddressChaining("aie-address-chaining", cl::Hidden,
                                     cl::init(true),
                                     cl::desc("Enable ptradd chaining."));
 
+cl::opt<bool> EnableGlobalPtrModOptimizer(
+    "aie-global-ptr-mod-opt", cl::Hidden, cl::init(true),
+    cl::desc("Enable global pointer modifier optimization."));
+
 cl::opt<bool>
     EnableStagedRA("aie-staged-ra", cl::Hidden, cl::init(true),
                    cl::desc("Enable multi-stage register allocation"));

llvm/lib/Target/AIE/AIECombine.td

@@ -237,12 +237,11 @@ def combine_upd_to_concat : GICombineRule<
   (apply [{ applyUpdToConcat(*${root}, MRI, B, ${matchinfo}); }])
 >;
 
-def load_store_increment_matchdata : GIDefMatchData<"AIELoadStoreCombineMatchData">;
 def combine_load_store_increment : GICombineRule <
-  (defs root:$root, load_store_increment_matchdata:$matchinfo),
+  (defs root:$root),
   (match (wip_match_opcode G_LOAD, G_ZEXTLOAD, G_SEXTLOAD, G_STORE):$root,
-          [{ return matchLdStInc(*${root}, MRI, ${matchinfo}, Helper, B.getTII()); }]),
-  (apply [{ applyLdStInc(*${root}, MRI, B, ${matchinfo}, Observer); }] )
+          [{ return matchLdStInc(*${root}, MRI, Helper, B.getTII(), GlobalCombiners); }]),
+  (apply [{ applyLdStInc(*${root}, MRI, Helper, B, Observer, GlobalCombiners); }] )
 >;
 
 def combine_add_vector_elt_undef : GICombineRule <
@@ -276,8 +275,18 @@ def combine_offset_load_store_share_ptradd : GICombineRule<
   (apply [{ applyOffsetLoadStoreSharePtrAdd(*${root}, MRI, B, ${matchinfo}); }])
 >;
 
+
+def combine_global_load_store_increment : GICombineRule <
+  (defs root:$root),
+  (match (wip_match_opcode G_LOAD, G_ZEXTLOAD, G_SEXTLOAD, G_STORE):$root,
+          [{ return matchGlobalPtrModOptimizer(*${root}, MRI, Helper, B.getTII(), GlobalCombiners); }]),
+  (apply [{ applyLdStInc(*${root}, MRI, Helper, B, Observer, GlobalCombiners); }] )>;
+
+
+
 def AIE2PostLegalizerCustomCombiner
-    : GICombiner<"AIE2PostLegalizerCustomCombinerImpl", [ combine_load_store_split,
+    : GICombiner<"AIE2PostLegalizerCustomCombinerImpl", [ combine_global_load_store_increment,
+                                                          combine_load_store_split,
                                                           ptr_add_immed_chain,
                                                           combine_load_store_increment,
                                                           combine_offset_load_store_ptradd,
@@ -290,9 +299,10 @@ def AIE2PostLegalizerCustomCombiner
 }
 
 def AIE2PPostLegalizerCustomCombiner
-    : GICombiner<"AIE2PPostLegalizerCustomCombinerImpl", [ combine_load_store_increment,
-                                                           ptr_add_immed_chain, 
-                                                           combine_offset_load_store_ptradd,
-                                                           combine_offset_load_store_share_ptradd,
-                                                           combine_add_vector_elt_undef ]> {
+    : GICombiner<"AIE2PPostLegalizerCustomCombinerImpl", [  combine_global_load_store_increment,
+                                                            combine_load_store_increment,
+                                                            ptr_add_immed_chain, 
+                                                            combine_offset_load_store_ptradd,
+                                                            combine_offset_load_store_share_ptradd,
+                                                            combine_add_vector_elt_undef ]> {
 }