Skip to content

Commit dc2ed00

Browse files
lukel97lukel97
authored
[RISCV] Handle non uimm5 VL constants in isVLKnownLE (#156639)
If a VL operand is > 31 then it will be materialized into an ADDI $x0, imm. We can reason about it by peeking at the virtual register definition which allows RISCVVectorPeephole and RISCVVLOptimizer to catch more cases. There's a separate issue with RISCVVLOptimizer where the materialized immediate may not always dominate the instruction we want to reduce the VL of, but this is left to another patch.
1 parent 4e5e65e commit dc2ed00

15 files changed

+87
-66
lines changed

‎llvm/lib/Target/RISCV/RISCVInstrInfo.cpp‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4796,8 +4796,22 @@ unsigned RISCV::getDestLog2EEW(const MCInstrDesc &Desc, unsigned Log2SEW) {
47964796
return Scaled;
47974797
}
47984798

4799-
/// Given two VL operands, do we know that LHS <= RHS?
4800-
bool RISCV::isVLKnownLE(const MachineOperand &LHS, const MachineOperand &RHS) {
4799+
static std::optional<int64_t> getEffectiveImm(const MachineOperand &MO,
4800+
const MachineRegisterInfo *MRI) {
4801+
assert(MO.isImm() || MO.getReg().isVirtual());
4802+
if (MO.isImm())
4803+
return MO.getImm();
4804+
const MachineInstr *Def = MRI->getVRegDef(MO.getReg());
4805+
int64_t Imm;
4806+
if (isLoadImm(Def, Imm))
4807+
return Imm;
4808+
return std::nullopt;
4809+
}
4810+
4811+
/// Given two VL operands, do we know that LHS <= RHS? Must be used in SSA form.
4812+
bool RISCV::isVLKnownLE(const MachineOperand &LHS, const MachineOperand &RHS,
4813+
const MachineRegisterInfo *MRI) {
4814+
assert(MRI->isSSA());
48014815
if (LHS.isReg() && RHS.isReg() && LHS.getReg().isVirtual() &&
48024816
LHS.getReg() == RHS.getReg())
48034817
return true;
@@ -4807,9 +4821,11 @@ bool RISCV::isVLKnownLE(const MachineOperand &LHS, const MachineOperand &RHS) {
48074821
return true;
48084822
if (LHS.isImm() && LHS.getImm() == RISCV::VLMaxSentinel)
48094823
return false;
4810-
if (!LHS.isImm() || !RHS.isImm())
4824+
std::optional<int64_t> LHSImm = getEffectiveImm(LHS, MRI),
4825+
RHSImm = getEffectiveImm(RHS, MRI);
4826+
if (!LHSImm || !RHSImm)
48114827
return false;
4812-
return LHS.getImm() <= RHS.getImm();
4828+
return LHSImm <= RHSImm;
48134829
}
48144830

48154831
namespace {

‎llvm/lib/Target/RISCV/RISCVInstrInfo.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -365,7 +365,8 @@ unsigned getDestLog2EEW(const MCInstrDesc &Desc, unsigned Log2SEW);
365365
static constexpr int64_t VLMaxSentinel = -1LL;
366366

367367
/// Given two VL operands, do we know that LHS <= RHS?
368-
bool isVLKnownLE(const MachineOperand &LHS, const MachineOperand &RHS);
368+
bool isVLKnownLE(const MachineOperand &LHS, const MachineOperand &RHS,
369+
const MachineRegisterInfo *MRI);
369370

370371
// Mask assignments for floating-point
371372
static constexpr unsigned FPMASK_Negative_Infinity = 0x001;

‎llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1379,7 +1379,7 @@ RISCVVLOptimizer::getMinimumVLForUser(const MachineOperand &UserOp) const {
13791379
assert(UserOp.getOperandNo() == UserMI.getNumExplicitDefs() &&
13801380
RISCVII::isFirstDefTiedToFirstUse(UserMI.getDesc()));
13811381
auto DemandedVL = DemandedVLs.lookup(&UserMI);
1382-
if (!DemandedVL || !RISCV::isVLKnownLE(*DemandedVL, VLOp)) {
1382+
if (!DemandedVL || !RISCV::isVLKnownLE(*DemandedVL, VLOp, MRI)) {
13831383
LLVM_DEBUG(dbgs() << " Abort because user is passthru in "
13841384
"instruction with demanded tail\n");
13851385
return std::nullopt;
@@ -1397,7 +1397,7 @@ RISCVVLOptimizer::getMinimumVLForUser(const MachineOperand &UserOp) const {
13971397
// requires.
13981398
if (auto DemandedVL = DemandedVLs.lookup(&UserMI)) {
13991399
assert(isCandidate(UserMI));
1400-
if (RISCV::isVLKnownLE(*DemandedVL, VLOp))
1400+
if (RISCV::isVLKnownLE(*DemandedVL, VLOp, MRI))
14011401
return DemandedVL;
14021402
}
14031403

@@ -1505,10 +1505,10 @@ RISCVVLOptimizer::checkUsers(const MachineInstr &MI) const {
15051505

15061506
// Use the largest VL among all the users. If we cannot determine this
15071507
// statically, then we cannot optimize the VL.
1508-
if (!CommonVL || RISCV::isVLKnownLE(*CommonVL, *VLOp)) {
1508+
if (!CommonVL || RISCV::isVLKnownLE(*CommonVL, *VLOp, MRI)) {
15091509
CommonVL = *VLOp;
15101510
LLVM_DEBUG(dbgs() << " User VL is: " << VLOp << "\n");
1511-
} else if (!RISCV::isVLKnownLE(*VLOp, *CommonVL)) {
1511+
} else if (!RISCV::isVLKnownLE(*VLOp, *CommonVL, MRI)) {
15121512
LLVM_DEBUG(dbgs() << " Abort because cannot determine a common VL\n");
15131513
return std::nullopt;
15141514
}
@@ -1570,7 +1570,7 @@ bool RISCVVLOptimizer::tryReduceVL(MachineInstr &MI) const {
15701570
CommonVL = VLMI->getOperand(RISCVII::getVLOpNum(VLMI->getDesc()));
15711571
}
15721572

1573-
if (!RISCV::isVLKnownLE(*CommonVL, VLOp)) {
1573+
if (!RISCV::isVLKnownLE(*CommonVL, VLOp, MRI)) {
15741574
LLVM_DEBUG(dbgs() << " Abort due to CommonVL not <= VLOp.\n");
15751575
return false;
15761576
}

‎llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,7 @@ bool RISCVVectorPeephole::tryToReduceVL(MachineInstr &MI) const {
177177

178178
MachineOperand &SrcVL =
179179
Src->getOperand(RISCVII::getVLOpNum(Src->getDesc()));
180-
if (VL.isIdenticalTo(SrcVL) || !RISCV::isVLKnownLE(VL, SrcVL))
180+
if (VL.isIdenticalTo(SrcVL) || !RISCV::isVLKnownLE(VL, SrcVL, MRI))
181181
continue;
182182

183183
if (!ensureDominates(VL, *Src))
@@ -440,7 +440,7 @@ bool RISCVVectorPeephole::convertSameMaskVMergeToVMv(MachineInstr &MI) {
440440
const MachineOperand &MIVL = MI.getOperand(RISCVII::getVLOpNum(MI.getDesc()));
441441
const MachineOperand &TrueVL =
442442
True->getOperand(RISCVII::getVLOpNum(True->getDesc()));
443-
if (!RISCV::isVLKnownLE(MIVL, TrueVL))
443+
if (!RISCV::isVLKnownLE(MIVL, TrueVL, MRI))
444444
return false;
445445

446446
// True's passthru needs to be equivalent to False
@@ -611,7 +611,7 @@ bool RISCVVectorPeephole::foldUndefPassthruVMV_V_V(MachineInstr &MI) {
611611
MachineOperand &SrcPolicy =
612612
Src->getOperand(RISCVII::getVecPolicyOpNum(Src->getDesc()));
613613

614-
if (RISCV::isVLKnownLE(MIVL, SrcVL))
614+
if (RISCV::isVLKnownLE(MIVL, SrcVL, MRI))
615615
SrcPolicy.setImm(SrcPolicy.getImm() | RISCVVType::TAIL_AGNOSTIC);
616616
}
617617

@@ -663,7 +663,7 @@ bool RISCVVectorPeephole::foldVMV_V_V(MachineInstr &MI) {
663663
// so we don't need to handle a smaller source VL here. However, the
664664
// user's VL may be larger
665665
MachineOperand &SrcVL = Src->getOperand(RISCVII::getVLOpNum(Src->getDesc()));
666-
if (!RISCV::isVLKnownLE(SrcVL, MI.getOperand(3)))
666+
if (!RISCV::isVLKnownLE(SrcVL, MI.getOperand(3), MRI))
667667
return false;
668668

669669
// If the new passthru doesn't dominate Src, try to move Src so it does.
@@ -684,7 +684,7 @@ bool RISCVVectorPeephole::foldVMV_V_V(MachineInstr &MI) {
684684
// If MI was tail agnostic and the VL didn't increase, preserve it.
685685
int64_t Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;
686686
if ((MI.getOperand(5).getImm() & RISCVVType::TAIL_AGNOSTIC) &&
687-
RISCV::isVLKnownLE(MI.getOperand(3), SrcVL))
687+
RISCV::isVLKnownLE(MI.getOperand(3), SrcVL, MRI))
688688
Policy |= RISCVVType::TAIL_AGNOSTIC;
689689
Src->getOperand(RISCVII::getVecPolicyOpNum(Src->getDesc())).setImm(Policy);
690690
}
@@ -775,9 +775,9 @@ bool RISCVVectorPeephole::foldVMergeToMask(MachineInstr &MI) const {
775775
True.getOperand(RISCVII::getVLOpNum(True.getDesc()));
776776

777777
MachineOperand MinVL = MachineOperand::CreateImm(0);
778-
if (RISCV::isVLKnownLE(TrueVL, VMergeVL))
778+
if (RISCV::isVLKnownLE(TrueVL, VMergeVL, MRI))
779779
MinVL = TrueVL;
780-
else if (RISCV::isVLKnownLE(VMergeVL, TrueVL))
780+
else if (RISCV::isVLKnownLE(VMergeVL, TrueVL, MRI))
781781
MinVL = VMergeVL;
782782
else
783783
return false;
@@ -797,7 +797,7 @@ bool RISCVVectorPeephole::foldVMergeToMask(MachineInstr &MI) const {
797797
// to the tail. In that case we always need to use tail undisturbed to
798798
// preserve them.
799799
uint64_t Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;
800-
if (!PassthruReg && RISCV::isVLKnownLE(VMergeVL, MinVL))
800+
if (!PassthruReg && RISCV::isVLKnownLE(VMergeVL, MinVL, MRI))
801801
Policy |= RISCVVType::TAIL_AGNOSTIC;
802802

803803
assert(RISCVII::hasVecPolicyOp(True.getDesc().TSFlags) &&

‎llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vadd-vp.ll‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -413,9 +413,10 @@ define <256 x i8> @vadd_vi_v258i8_unmasked(<256 x i8> %va, i32 zeroext %evl) {
413413
define <256 x i8> @vadd_vi_v258i8_evl129(<256 x i8> %va, <256 x i1> %m) {
414414
; CHECK-LABEL: vadd_vi_v258i8_evl129:
415415
; CHECK: # %bb.0:
416-
; CHECK-NEXT: li a1, 128
417-
; CHECK-NEXT: vsetvli zero, a1, e8, m8, ta, ma
416+
; CHECK-NEXT: vsetivli zero, 1, e8, m8, ta, ma
418417
; CHECK-NEXT: vlm.v v24, (a0)
418+
; CHECK-NEXT: li a0, 128
419+
; CHECK-NEXT: vsetvli zero, a0, e8, m8, ta, ma
419420
; CHECK-NEXT: vadd.vi v8, v8, -1, v0.t
420421
; CHECK-NEXT: vmv1r.v v0, v24
421422
; CHECK-NEXT: vsetivli zero, 1, e8, m8, ta, ma

‎llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmax-vp.ll‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -321,9 +321,10 @@ define <256 x i8> @vmax_vx_v258i8_unmasked(<256 x i8> %va, i8 %b, i32 zeroext %e
321321
define <256 x i8> @vmax_vx_v258i8_evl129(<256 x i8> %va, i8 %b, <256 x i1> %m) {
322322
; CHECK-LABEL: vmax_vx_v258i8_evl129:
323323
; CHECK: # %bb.0:
324-
; CHECK-NEXT: li a2, 128
325-
; CHECK-NEXT: vsetvli zero, a2, e8, m8, ta, ma
324+
; CHECK-NEXT: vsetivli zero, 1, e8, m8, ta, ma
326325
; CHECK-NEXT: vlm.v v24, (a1)
326+
; CHECK-NEXT: li a1, 128
327+
; CHECK-NEXT: vsetvli zero, a1, e8, m8, ta, ma
327328
; CHECK-NEXT: vmax.vx v8, v8, a0, v0.t
328329
; CHECK-NEXT: vmv1r.v v0, v24
329330
; CHECK-NEXT: vsetivli zero, 1, e8, m8, ta, ma

‎llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmaxu-vp.ll‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -320,9 +320,10 @@ define <256 x i8> @vmaxu_vx_v258i8_unmasked(<256 x i8> %va, i8 %b, i32 zeroext %
320320
define <256 x i8> @vmaxu_vx_v258i8_evl129(<256 x i8> %va, i8 %b, <256 x i1> %m) {
321321
; CHECK-LABEL: vmaxu_vx_v258i8_evl129:
322322
; CHECK: # %bb.0:
323-
; CHECK-NEXT: li a2, 128
324-
; CHECK-NEXT: vsetvli zero, a2, e8, m8, ta, ma
323+
; CHECK-NEXT: vsetivli zero, 1, e8, m8, ta, ma
325324
; CHECK-NEXT: vlm.v v24, (a1)
325+
; CHECK-NEXT: li a1, 128
326+
; CHECK-NEXT: vsetvli zero, a1, e8, m8, ta, ma
326327
; CHECK-NEXT: vmaxu.vx v8, v8, a0, v0.t
327328
; CHECK-NEXT: vmv1r.v v0, v24
328329
; CHECK-NEXT: vsetivli zero, 1, e8, m8, ta, ma

‎llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vmin-vp.ll‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -321,9 +321,10 @@ define <256 x i8> @vmin_vx_v258i8_unmasked(<256 x i8> %va, i8 %b, i32 zeroext %e
321321
define <256 x i8> @vmin_vx_v258i8_evl129(<256 x i8> %va, i8 %b, <256 x i1> %m) {
322322
; CHECK-LABEL: vmin_vx_v258i8_evl129:
323323
; CHECK: # %bb.0:
324-
; CHECK-NEXT: li a2, 128
325-
; CHECK-NEXT: vsetvli zero, a2, e8, m8, ta, ma
324+
; CHECK-NEXT: vsetivli zero, 1, e8, m8, ta, ma
326325
; CHECK-NEXT: vlm.v v24, (a1)
326+
; CHECK-NEXT: li a1, 128
327+
; CHECK-NEXT: vsetvli zero, a1, e8, m8, ta, ma
327328
; CHECK-NEXT: vmin.vx v8, v8, a0, v0.t
328329
; CHECK-NEXT: vmv1r.v v0, v24
329330
; CHECK-NEXT: vsetivli zero, 1, e8, m8, ta, ma

‎llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vminu-vp.ll‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -320,9 +320,10 @@ define <256 x i8> @vminu_vx_v258i8_unmasked(<256 x i8> %va, i8 %b, i32 zeroext %
320320
define <256 x i8> @vminu_vx_v258i8_evl129(<256 x i8> %va, i8 %b, <256 x i1> %m) {
321321
; CHECK-LABEL: vminu_vx_v258i8_evl129:
322322
; CHECK: # %bb.0:
323-
; CHECK-NEXT: li a2, 128
324-
; CHECK-NEXT: vsetvli zero, a2, e8, m8, ta, ma
323+
; CHECK-NEXT: vsetivli zero, 1, e8, m8, ta, ma
325324
; CHECK-NEXT: vlm.v v24, (a1)
325+
; CHECK-NEXT: li a1, 128
326+
; CHECK-NEXT: vsetvli zero, a1, e8, m8, ta, ma
326327
; CHECK-NEXT: vminu.vx v8, v8, a0, v0.t
327328
; CHECK-NEXT: vmv1r.v v0, v24
328329
; CHECK-NEXT: vsetivli zero, 1, e8, m8, ta, ma

‎llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vsadd-vp.ll‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -422,9 +422,10 @@ define <256 x i8> @vsadd_vi_v258i8_unmasked(<256 x i8> %va, i32 zeroext %evl) {
422422
define <256 x i8> @vsadd_vi_v258i8_evl129(<256 x i8> %va, <256 x i1> %m) {
423423
; CHECK-LABEL: vsadd_vi_v258i8_evl129:
424424
; CHECK: # %bb.0:
425-
; CHECK-NEXT: li a1, 128
426-
; CHECK-NEXT: vsetvli zero, a1, e8, m8, ta, ma
425+
; CHECK-NEXT: vsetivli zero, 1, e8, m8, ta, ma
427426
; CHECK-NEXT: vlm.v v24, (a0)
427+
; CHECK-NEXT: li a0, 128
428+
; CHECK-NEXT: vsetvli zero, a0, e8, m8, ta, ma
428429
; CHECK-NEXT: vsadd.vi v8, v8, -1, v0.t
429430
; CHECK-NEXT: vmv1r.v v0, v24
430431
; CHECK-NEXT: vsetivli zero, 1, e8, m8, ta, ma

0 commit comments

Comments
 (0)