[𝘀𝗽𝗿] changes introduced through rebase

Created using spr 1.3.5

[skip ci]

Author: aemerson

llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h

@@ -22,6 +22,7 @@
 
 #include "llvm/CodeGen/GlobalISel/CallLowering.h"
 #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
+#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
 #include "llvm/CodeGen/RuntimeLibcallUtil.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
 
@@ -297,9 +298,12 @@ class LegalizerHelper {
   MachineInstrBuilder createStackTemporary(TypeSize Bytes, Align Alignment,
                                            MachinePointerInfo &PtrInfo);
 
-  /// Create a store of \p Val to a stack temporary and return a load of the
-  /// same value as type \p DestVT.
-  MachineInstrBuilder createStackStoreLoad(Register Val, LLT DstTy);
+  /// Create a store of \p Val to a stack temporary and return a load as the
+  /// same type as \p Res.
+  MachineInstrBuilder createStackStoreLoad(const DstOp &Res, const SrcOp &Val);
+
+  /// Given a store of a boolean vector, scalarize it.
+  LegalizeResult scalarizeVectorBooleanStore(GStore &MI);
 
   /// Get a pointer to vector element \p Index located in memory for a vector of
   /// type \p VecTy starting at a base address of \p VecPtr. If \p Index is out

llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp

@@ -3023,6 +3023,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
       return UnableToLegalize;
 
     LLT Ty = MRI.getType(MI.getOperand(0).getReg());
+    assert(!Ty.isPointerOrPointerVector() && "Can't widen type");
     if (!Ty.isScalar()) {
       // We need to widen the vector element type.
       Observer.changingInstr(MI);
@@ -4142,40 +4143,8 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerStore(GStore &StoreMI) {
   }
 
   if (MemTy.isVector()) {
-    LLT MemScalarTy = MemTy.getElementType();
-    if (MemTy != SrcTy) {
-      if (!MemScalarTy.isByteSized()) {
-        // We need to build an integer scalar of the vector bit pattern.
-        // It's not legal for us to add padding when storing a vector.
-        unsigned NumBits = MemTy.getSizeInBits();
-        LLT IntTy = LLT::scalar(NumBits);
-        auto CurrVal = MIRBuilder.buildConstant(IntTy, 0);
-        LLT IdxTy = getLLTForMVT(TLI.getVectorIdxTy(MF.getDataLayout()));
-
-        for (unsigned I = 0, E = MemTy.getNumElements(); I < E; ++I) {
-          auto Elt = MIRBuilder.buildExtractVectorElement(
-              SrcTy.getElementType(), SrcReg,
-              MIRBuilder.buildConstant(IdxTy, I));
-          auto Trunc = MIRBuilder.buildTrunc(MemScalarTy, Elt);
-          auto ZExt = MIRBuilder.buildZExt(IntTy, Trunc);
-          unsigned ShiftIntoIdx = MF.getDataLayout().isBigEndian()
-                                      ? (MemTy.getNumElements() - 1) - I
-                                      : I;
-          auto ShiftAmt = MIRBuilder.buildConstant(
-              IntTy, ShiftIntoIdx * MemScalarTy.getSizeInBits());
-          auto Shifted = MIRBuilder.buildShl(IntTy, ZExt, ShiftAmt);
-          CurrVal = MIRBuilder.buildOr(IntTy, CurrVal, Shifted);
-        }
-        auto PtrInfo = MMO.getPointerInfo();
-        auto *NewMMO = MF.getMachineMemOperand(&MMO, PtrInfo, IntTy);
-        MIRBuilder.buildStore(CurrVal, PtrReg, *NewMMO);
-        StoreMI.eraseFromParent();
-        return Legalized;
-      }
-
-      // FIXME: implement simple scalarization.
-      return UnableToLegalize;
-    }
+    if (MemTy != SrcTy)
+      return scalarizeVectorBooleanStore(StoreMI);
 
     // TODO: We can do better than scalarizing the vector and at least split it
     // in half.
@@ -4230,6 +4199,50 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerStore(GStore &StoreMI) {
   return Legalized;
 }
 
+LegalizerHelper::LegalizeResult
+LegalizerHelper::scalarizeVectorBooleanStore(GStore &StoreMI) {
+  Register SrcReg = StoreMI.getValueReg();
+  Register PtrReg = StoreMI.getPointerReg();
+  LLT SrcTy = MRI.getType(SrcReg);
+  MachineMemOperand &MMO = **StoreMI.memoperands_begin();
+  LLT MemTy = MMO.getMemoryType();
+  LLT MemScalarTy = MemTy.getElementType();
+  MachineFunction &MF = MIRBuilder.getMF();
+
+  assert(SrcTy.isVector() && "Expect a vector store type");
+
+  if (!MemScalarTy.isByteSized()) {
+    // We need to build an integer scalar of the vector bit pattern.
+    // It's not legal for us to add padding when storing a vector.
+    unsigned NumBits = MemTy.getSizeInBits();
+    LLT IntTy = LLT::scalar(NumBits);
+    auto CurrVal = MIRBuilder.buildConstant(IntTy, 0);
+    LLT IdxTy = getLLTForMVT(TLI.getVectorIdxTy(MF.getDataLayout()));
+
+    for (unsigned I = 0, E = MemTy.getNumElements(); I < E; ++I) {
+      auto Elt = MIRBuilder.buildExtractVectorElement(
+          SrcTy.getElementType(), SrcReg, MIRBuilder.buildConstant(IdxTy, I));
+      auto Trunc = MIRBuilder.buildTrunc(MemScalarTy, Elt);
+      auto ZExt = MIRBuilder.buildZExt(IntTy, Trunc);
+      unsigned ShiftIntoIdx = MF.getDataLayout().isBigEndian()
+                                  ? (MemTy.getNumElements() - 1) - I
+                                  : I;
+      auto ShiftAmt = MIRBuilder.buildConstant(
+          IntTy, ShiftIntoIdx * MemScalarTy.getSizeInBits());
+      auto Shifted = MIRBuilder.buildShl(IntTy, ZExt, ShiftAmt);
+      CurrVal = MIRBuilder.buildOr(IntTy, CurrVal, Shifted);
+    }
+    auto PtrInfo = MMO.getPointerInfo();
+    auto *NewMMO = MF.getMachineMemOperand(&MMO, PtrInfo, IntTy);
+    MIRBuilder.buildStore(CurrVal, PtrReg, *NewMMO);
+    StoreMI.eraseFromParent();
+    return Legalized;
+  }
+
+  // TODO: implement simple scalarization.
+  return UnableToLegalize;
+}
+
 LegalizerHelper::LegalizeResult
 LegalizerHelper::bitcast(MachineInstr &MI, unsigned TypeIdx, LLT CastTy) {
   switch (MI.getOpcode()) {
@@ -4695,18 +4708,18 @@ LegalizerHelper::createStackTemporary(TypeSize Bytes, Align Alignment,
   return MIRBuilder.buildFrameIndex(FramePtrTy, FrameIdx);
 }
 
-MachineInstrBuilder LegalizerHelper::createStackStoreLoad(Register Val,
-                                                          LLT DstTy) {
-  LLT SrcTy = MRI.getType(Val);
-  Align StackTypeAlign = getStackTemporaryAlignment(SrcTy);
+MachineInstrBuilder LegalizerHelper::createStackStoreLoad(const DstOp &Res,
+                                                          const SrcOp &Val) {
+  LLT SrcTy = Val.getLLTTy(MRI);
+  Align StackTypeAlign =
+      std::max(getStackTemporaryAlignment(SrcTy),
+               getStackTemporaryAlignment(Res.getLLTTy(MRI)));
   MachinePointerInfo PtrInfo;
   auto StackTemp =
       createStackTemporary(SrcTy.getSizeInBytes(), StackTypeAlign, PtrInfo);
 
   MIRBuilder.buildStore(Val, StackTemp, PtrInfo, StackTypeAlign);
-  return MIRBuilder.buildLoad(
-      DstTy, StackTemp, PtrInfo,
-      std::min(StackTypeAlign, getStackTemporaryAlignment(DstTy)));
+  return MIRBuilder.buildLoad(Res, StackTemp, PtrInfo, StackTypeAlign);
 }
 
 static Register clampVectorIndex(MachineIRBuilder &B, Register IdxReg,

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

@@ -1430,8 +1430,7 @@ bool AArch64LegalizerInfo::legalizeBitcast(MachineInstr &MI,
       SrcTy.getElementType() != LLT::scalar(1))
     return false;
 
-  auto Load = Helper.createStackStoreLoad(SrcReg, DstTy);
-  Helper.MIRBuilder.buildCopy(DstReg, Load.getReg(0));
+  Helper.createStackStoreLoad(DstReg, SrcReg);
   MI.eraseFromParent();
   return true;
 }

llvm/test/CodeGen/AArch64/GlobalISel/legalize-bitcast.mir

@@ -108,8 +108,7 @@ body:             |
     ; CHECK-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[OR6]], [[SHL7]]
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[OR7]](s32)
     ; CHECK-NEXT: G_STORE [[TRUNC]](s8), [[FRAME_INDEX]](p0) :: (store (s8) into %stack.0)
-    ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX]](p0) :: (load (s8) from %stack.0)
-    ; CHECK-NEXT: %bc:_(s8) = COPY [[LOAD]](s8)
+    ; CHECK-NEXT: %bc:_(s8) = G_LOAD [[FRAME_INDEX]](p0) :: (load (s8) from %stack.0)
     ; CHECK-NEXT: %ext:_(s32) = G_ANYEXT %bc(s8)
     ; CHECK-NEXT: $w0 = COPY %ext(s32)
     ; CHECK-NEXT: RET_ReallyLR implicit $w0