[X86][GlobalISel] Support addr matching in SDAG pattern

addr matching was the only gatekeeper for starting selecting G_LOAD and
G_STORE using SDAG patterns.

* Replace getLoadStoreOp with getPtrLoadStoreOp in Load/Store selector
  as GlobalISel matcher or emitter can't map the pointer type into
  i32/i64 types used in SDAG patterns for pointers. So the load and
  store selection of pointers is still manual.
* X86SelectAddress now is used for both: pattern matching and manual
  selection. As a result it accumulates all the code that previously was
  distrubuted among different selection functions.
* Introduce a complex rederer `gi_addr` for `addr`. In this patch only
  existing functionality has been implemented. The renderer's name is
  the same as in SDAG: `selectAddr`.
* Since truncating stores are not supported, we custom legalize them by
  matching types of store and MMO.
* Introduce a ConstantPool flag to X86AddressMode because otherwise we
  need to introduce a GlobalISel copy for X86ISelAddressMode.

Author: e-kud

llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h

@@ -66,6 +66,9 @@ class GMemOperation : public GenericMachineInstr {
   /// memory operations can't be reordered.
   bool isUnordered() const { return getMMO().isUnordered(); }
 
+  /// Return the minimum known alignment in bytes of the actual memory
+  /// reference.
+  Align getAlign() const { return getMMO().getAlign(); }
   /// Returns the size in bytes of the memory access.
   LocationSize getMemSize() const { return getMMO().getSize(); }
   /// Returns the size in bits of the memory access.

llvm/lib/Target/X86/GISel/X86InstructionSelector.cpp

@@ -73,6 +73,9 @@ class X86InstructionSelector : public InstructionSelector {
   // TODO: remove after supported by Tablegen-erated instruction selection.
   unsigned getLoadStoreOp(const LLT &Ty, const RegisterBank &RB, unsigned Opc,
                           Align Alignment) const;
+  // TODO: remove once p0<->i32/i64 matching is available
+  unsigned getPtrLoadStoreOp(const LLT &Ty, const RegisterBank &RB,
+                             unsigned Opc) const;
 
   bool selectLoadStoreOp(MachineInstr &I, MachineRegisterInfo &MRI,
                          MachineFunction &MF) const;
@@ -119,6 +122,8 @@ class X86InstructionSelector : public InstructionSelector {
   bool selectSelect(MachineInstr &I, MachineRegisterInfo &MRI,
                     MachineFunction &MF) const;
 
+  ComplexRendererFns selectAddr(MachineOperand &Root) const;
+
   // emit insert subreg instruction and insert it before MachineInstr &I
   bool emitInsertSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I,
                         MachineRegisterInfo &MRI, MachineFunction &MF) const;
@@ -445,6 +450,17 @@ bool X86InstructionSelector::select(MachineInstr &I) {
   return false;
 }
 
+unsigned X86InstructionSelector::getPtrLoadStoreOp(const LLT &Ty,
+                                                   const RegisterBank &RB,
+                                                   unsigned Opc) const {
+  bool Isload = (Opc == TargetOpcode::G_LOAD);
+  if (Ty == LLT::pointer(0, 32) && X86::GPRRegBankID == RB.getID())
+    return Isload ? X86::MOV32rm : X86::MOV32mr;
+  if (Ty == LLT::pointer(0, 64) && X86::GPRRegBankID == RB.getID())
+    return Isload ? X86::MOV64rm : X86::MOV64mr;
+  return Opc;
+}
+
 unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
                                                 const RegisterBank &RB,
                                                 unsigned Opc,
@@ -460,7 +476,7 @@ unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
   } else if (Ty == LLT::scalar(16)) {
     if (X86::GPRRegBankID == RB.getID())
       return Isload ? X86::MOV16rm : X86::MOV16mr;
-  } else if (Ty == LLT::scalar(32) || Ty == LLT::pointer(0, 32)) {
+  } else if (Ty == LLT::scalar(32)) {
     if (X86::GPRRegBankID == RB.getID())
       return Isload ? X86::MOV32rm : X86::MOV32mr;
     if (X86::VECRRegBankID == RB.getID())
@@ -472,7 +488,7 @@ unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
                                    X86::MOVSSmr);
     if (X86::PSRRegBankID == RB.getID())
       return Isload ? X86::LD_Fp32m : X86::ST_Fp32m;
-  } else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) {
+  } else if (Ty == LLT::scalar(64)) {
     if (X86::GPRRegBankID == RB.getID())
       return Isload ? X86::MOV64rm : X86::MOV64mr;
     if (X86::VECRRegBankID == RB.getID())
@@ -530,30 +546,75 @@ unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
 }
 
 // Fill in an address from the given instruction.
-static void X86SelectAddress(const MachineInstr &I,
+static bool X86SelectAddress(MachineInstr &I, const X86TargetMachine &TM,
                              const MachineRegisterInfo &MRI,
-                             X86AddressMode &AM) {
-  assert(I.getOperand(0).isReg() && "unsupported opperand.");
+                             const X86Subtarget &STI, X86AddressMode &AM) {
+  assert(I.getOperand(0).isReg() && "unsupported operand.");
   assert(MRI.getType(I.getOperand(0).getReg()).isPointer() &&
          "unsupported type.");
 
-  if (I.getOpcode() == TargetOpcode::G_PTR_ADD) {
+  switch (I.getOpcode()) {
+  default:
+    break;
+  case TargetOpcode::G_FRAME_INDEX:
+    AM.Base.FrameIndex = I.getOperand(1).getIndex();
+    AM.BaseType = X86AddressMode::FrameIndexBase;
+    return true;
+  case TargetOpcode::G_PTR_ADD: {
     if (auto COff = getIConstantVRegSExtVal(I.getOperand(2).getReg(), MRI)) {
       int64_t Imm = *COff;
       if (isInt<32>(Imm)) { // Check for displacement overflow.
         AM.Disp = static_cast<int32_t>(Imm);
         AM.Base.Reg = I.getOperand(1).getReg();
-        return;
+        return true;
       }
     }
-  } else if (I.getOpcode() == TargetOpcode::G_FRAME_INDEX) {
-    AM.Base.FrameIndex = I.getOperand(1).getIndex();
-    AM.BaseType = X86AddressMode::FrameIndexBase;
-    return;
+    break;
   }
+  case TargetOpcode::G_GLOBAL_VALUE: {
+    auto GV = I.getOperand(1).getGlobal();
+    if (GV->isThreadLocal()) {
+      return false; // TODO: we don't support TLS yet.
+    }
+    // Can't handle alternate code models yet.
+    if (TM.getCodeModel() != CodeModel::Small)
+      return false;
+    AM.GV = GV;
+    AM.GVOpFlags = STI.classifyGlobalReference(GV);
+
+    // TODO: The ABI requires an extra load. not supported yet.
+    if (isGlobalStubReference(AM.GVOpFlags))
+      return false;
 
+    // TODO: This reference is relative to the pic base. not supported yet.
+    if (isGlobalRelativeToPICBase(AM.GVOpFlags))
+      return false;
+
+    if (STI.isPICStyleRIPRel()) {
+      // Use rip-relative addressing.
+      assert(AM.Base.Reg == 0 && AM.IndexReg == 0);
+      AM.Base.Reg = X86::RIP;
+    }
+    return true;
+  }
+  case TargetOpcode::G_CONSTANT_POOL: {
+    // TODO: Need a separate move for Large model
+    if (TM.getCodeModel() == CodeModel::Large)
+      return false;
+
+    AM.GVOpFlags = STI.classifyLocalReference(nullptr);
+    if (AM.GVOpFlags == X86II::MO_GOTOFF)
+      AM.Base.Reg = STI.getInstrInfo()->getGlobalBaseReg(I.getMF());
+    else if (STI.is64Bit())
+      AM.Base.Reg = X86::RIP;
+    AM.CP = true;
+    AM.Disp = I.getOperand(1).getIndex();
+    return true;
+  }
+  }
   // Default behavior.
   AM.Base.Reg = I.getOperand(0).getReg();
+  return true;
 }
 
 bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I,
@@ -586,36 +647,18 @@ bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I,
     }
   }
 
-  unsigned NewOpc = getLoadStoreOp(Ty, RB, Opc, MemOp.getAlign());
+  unsigned NewOpc = getPtrLoadStoreOp(Ty, RB, Opc);
   if (NewOpc == Opc)
     return false;
 
   I.setDesc(TII.get(NewOpc));
   MachineInstrBuilder MIB(MF, I);
-  const MachineInstr *Ptr = MRI.getVRegDef(I.getOperand(1).getReg());
-
-  if (Ptr->getOpcode() == TargetOpcode::G_CONSTANT_POOL) {
-    assert(Opc == TargetOpcode::G_LOAD &&
-           "Only G_LOAD from constant pool is expected");
-    // TODO: Need a separate move for Large model
-    if (TM.getCodeModel() == CodeModel::Large)
-      return false;
-
-    unsigned char OpFlag = STI.classifyLocalReference(nullptr);
-    unsigned PICBase = 0;
-    if (OpFlag == X86II::MO_GOTOFF)
-      PICBase = TII.getGlobalBaseReg(&MF);
-    else if (STI.is64Bit())
-      PICBase = X86::RIP;
-
-    I.removeOperand(1);
-    addConstantPoolReference(MIB, Ptr->getOperand(1).getIndex(), PICBase,
-                             OpFlag);
-    return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
-  }
+  MachineInstr *Ptr = MRI.getVRegDef(I.getOperand(1).getReg());
 
   X86AddressMode AM;
-  X86SelectAddress(*Ptr, MRI, AM);
+  if (!X86SelectAddress(*Ptr, TM, MRI, STI, AM))
+    return false;
+
   if (Opc == TargetOpcode::G_LOAD) {
     I.removeOperand(1);
     addFullAddress(MIB, AM);
@@ -673,33 +716,10 @@ bool X86InstructionSelector::selectGlobalValue(MachineInstr &I,
   assert((I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE) &&
          "unexpected instruction");
 
-  auto GV = I.getOperand(1).getGlobal();
-  if (GV->isThreadLocal()) {
-    return false; // TODO: we don't support TLS yet.
-  }
-
-  // Can't handle alternate code models yet.
-  if (TM.getCodeModel() != CodeModel::Small)
-    return false;
-
   X86AddressMode AM;
-  AM.GV = GV;
-  AM.GVOpFlags = STI.classifyGlobalReference(GV);
-
-  // TODO: The ABI requires an extra load. not supported yet.
-  if (isGlobalStubReference(AM.GVOpFlags))
+  if (!X86SelectAddress(I, TM, MRI, STI, AM))
     return false;
 
-  // TODO: This reference is relative to the pic base. not supported yet.
-  if (isGlobalRelativeToPICBase(AM.GVOpFlags))
-    return false;
-
-  if (STI.isPICStyleRIPRel()) {
-    // Use rip-relative addressing.
-    assert(AM.Base.Reg == 0 && AM.IndexReg == 0);
-    AM.Base.Reg = X86::RIP;
-  }
-
   const Register DefReg = I.getOperand(0).getReg();
   LLT Ty = MRI.getType(DefReg);
   unsigned NewOpc = getLeaOP(Ty, STI);
@@ -1880,6 +1900,49 @@ bool X86InstructionSelector::selectSelect(MachineInstr &I,
   return true;
 }
 
+InstructionSelector::ComplexRendererFns
+X86InstructionSelector::selectAddr(MachineOperand &Root) const {
+  MachineInstr *MI = Root.getParent();
+  MachineIRBuilder MIRBuilder(*MI);
+
+  MachineRegisterInfo &MRI = MI->getMF()->getRegInfo();
+  MachineInstr *Ptr = MRI.getVRegDef(Root.getReg());
+  X86AddressMode AM;
+  X86SelectAddress(*Ptr, TM, MRI, STI, AM);
+
+  if (AM.Scale != 1)
+    return std::nullopt;
+
+  if (AM.IndexReg)
+    return std::nullopt;
+
+  return {// Base
+          {[=](MachineInstrBuilder &MIB) {
+             if (AM.BaseType == X86AddressMode::RegBase)
+               MIB.addUse(AM.Base.Reg);
+             else {
+               assert(AM.BaseType == X86AddressMode::FrameIndexBase &&
+                      "Unknown type of address base");
+               MIB.addFrameIndex(AM.Base.FrameIndex);
+             }
+           },
+           // Scale
+           [=](MachineInstrBuilder &MIB) { MIB.addImm(AM.Scale); },
+           // Index
+           [=](MachineInstrBuilder &MIB) { MIB.addUse(0); },
+           // Disp
+           [=](MachineInstrBuilder &MIB) {
+             if (AM.GV)
+               MIB.addGlobalAddress(AM.GV, AM.Disp, AM.GVOpFlags);
+             else if (AM.CP)
+               MIB.addConstantPoolIndex(AM.Disp, 0, AM.GVOpFlags);
+             else
+               MIB.addImm(AM.Disp);
+           },
+           // Segment
+           [=](MachineInstrBuilder &MIB) { MIB.addUse(0); }}};
+}
+
 InstructionSelector *
 llvm::createX86InstructionSelector(const X86TargetMachine &TM,
                                    const X86Subtarget &Subtarget,

llvm/lib/Target/X86/GISel/X86LegalizerInfo.cpp

@@ -368,22 +368,16 @@ X86LegalizerInfo::X86LegalizerInfo(const X86Subtarget &STI,
   // load/store: add more corner cases
   for (unsigned Op : {G_LOAD, G_STORE}) {
     auto &Action = getActionDefinitionsBuilder(Op);
-    Action.legalForTypesWithMemDesc({{s8, p0, s1, 1},
-                                     {s8, p0, s8, 1},
-                                     {s16, p0, s8, 1},
+    Action.legalForTypesWithMemDesc({{s8, p0, s8, 1},
                                      {s16, p0, s16, 1},
-                                     {s32, p0, s8, 1},
-                                     {s32, p0, s16, 1},
                                      {s32, p0, s32, 1},
                                      {s80, p0, s80, 1},
                                      {p0, p0, p0, 1},
                                      {v4s8, p0, v4s8, 1}});
     if (Is64Bit)
-      Action.legalForTypesWithMemDesc({{s64, p0, s8, 1},
-                                       {s64, p0, s16, 1},
-                                       {s64, p0, s32, 1},
-                                       {s64, p0, s64, 1},
+      Action.legalForTypesWithMemDesc({{s64, p0, s64, 1},
                                        {v2s32, p0, v2s32, 1}});
+
     if (HasSSE1)
       Action.legalForTypesWithMemDesc({{v4s32, p0, v4s32, 1}});
     if (HasSSE2)
@@ -402,6 +396,22 @@ X86LegalizerInfo::X86LegalizerInfo(const X86Subtarget &STI,
                                        {v32s16, p0, v32s16, 1},
                                        {v16s32, p0, v16s32, 1},
                                        {v8s64, p0, v8s64, 1}});
+
+    // X86 supports extending loads but not stores for GPRs
+    if (Op == G_LOAD) {
+      Action.legalForTypesWithMemDesc({{s8, p0, s1, 1},
+                                       {s16, p0, s8, 1},
+                                       {s32, p0, s8, 1},
+                                       {s32, p0, s16, 1}});
+      if (Is64Bit)
+        Action.legalForTypesWithMemDesc({{s64, p0, s8, 1},
+                                         {s64, p0, s16, 1},
+                                         {s64, p0, s32, 1}});
+    } else {
+      Action.customIf([=](const LegalityQuery &Query) {
+        return Query.Types[0] != Query.MMODescrs[0].MemoryTy;
+      });
+    }
     Action.widenScalarToNextPow2(0, /*Min=*/8)
         .clampScalar(0, s8, sMaxScalar)
         .scalarize(0);
@@ -655,6 +665,8 @@ bool X86LegalizerInfo::legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI,
     return legalizeFPTOUI(MI, MRI, Helper);
   case TargetOpcode::G_UITOFP:
     return legalizeUITOFP(MI, MRI, Helper);
+  case TargetOpcode::G_STORE:
+    return legalizeNarrowingStore(MI, MRI, Helper);
   }
   llvm_unreachable("expected switch to return");
 }
@@ -749,6 +761,22 @@ bool X86LegalizerInfo::legalizeUITOFP(MachineInstr &MI,
   return false;
 }
 
+bool X86LegalizerInfo::legalizeNarrowingStore(MachineInstr &MI,
+                                              MachineRegisterInfo &MRI,
+                                              LegalizerHelper &Helper) const {
+  auto &Store = cast<GStore>(MI);
+  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
+  MachineMemOperand &MMO = **Store.memoperands_begin();
+  MachineFunction &MF = MIRBuilder.getMF();
+  LLT ValTy = MRI.getType(Store.getValueReg());
+  auto *NewMMO = MF.getMachineMemOperand(&MMO, MMO.getPointerInfo(), ValTy);
+
+  Helper.Observer.changingInstr(Store);
+  Store.setMemRefs(MF, {NewMMO});
+  Helper.Observer.changedInstr(Store);
+  return true;
+}
+
 bool X86LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
                                          MachineInstr &MI) const {
   return true;

llvm/lib/Target/X86/GISel/X86LegalizerInfo.h

@@ -45,6 +45,9 @@ class X86LegalizerInfo : public LegalizerInfo {
 
   bool legalizeUITOFP(MachineInstr &MI, MachineRegisterInfo &MRI,
                       LegalizerHelper &Helper) const;
+
+  bool legalizeNarrowingStore(MachineInstr &MI, MachineRegisterInfo &MRI,
+                              LegalizerHelper &Helper) const;
 };
 } // namespace llvm
 #endif

llvm/lib/Target/X86/X86InstrBuilder.h

@@ -55,10 +55,11 @@ struct X86AddressMode {
   int Disp;
   const GlobalValue *GV;
   unsigned GVOpFlags;
+  bool CP;
 
   X86AddressMode()
-    : BaseType(RegBase), Scale(1), IndexReg(0), Disp(0), GV(nullptr),
-      GVOpFlags(0) {
+      : BaseType(RegBase), Scale(1), IndexReg(0), Disp(0), GV(nullptr),
+        GVOpFlags(0), CP(false) {
     Base.Reg = 0;
   }