Skip to content

Commit fc5c5a9

Browse files
jyli0116jyli0116
authored
[GlobalISel] Allow expansion of srem by constant in prelegalizer (#148845)
This patch allows srem by a constant to be expanded more efficiently to avoid the need for expensive sdiv instructions. This is the last part of the patches which fixes #118090
1 parent 4166df2 commit fc5c5a9

File tree

5 files changed

+449
-616
lines changed

5 files changed

+449
-616
lines changed

‎llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -700,18 +700,19 @@ class CombinerHelper {
700700
/// Given an G_UDIV \p MI or G_UREM \p MI expressing a divide by constant,
701701
/// return an expression that implements it by multiplying by a magic number.
702702
/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
703-
MachineInstr *buildUDivorURemUsingMul(MachineInstr &MI) const;
703+
MachineInstr *buildUDivOrURemUsingMul(MachineInstr &MI) const;
704704
/// Combine G_UDIV or G_UREM by constant into a multiply by magic constant.
705-
bool matchUDivorURemByConst(MachineInstr &MI) const;
706-
void applyUDivorURemByConst(MachineInstr &MI) const;
707-
708-
/// Given an G_SDIV \p MI expressing a signed divide by constant, return an
709-
/// expression that implements it by multiplying by a magic number.
710-
/// Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's Guide".
711-
MachineInstr *buildSDivUsingMul(MachineInstr &MI) const;
712-
/// Combine G_SDIV by constant into a multiply by magic constant.
713-
bool matchSDivByConst(MachineInstr &MI) const;
714-
void applySDivByConst(MachineInstr &MI) const;
705+
bool matchUDivOrURemByConst(MachineInstr &MI) const;
706+
void applyUDivOrURemByConst(MachineInstr &MI) const;
707+
708+
/// Given an G_SDIV \p MI or G_SREM \p MI expressing a signed divide by
709+
/// constant, return an expression that implements it by multiplying by a
710+
/// magic number. Ref: "Hacker's Delight" or "The PowerPC Compiler Writer's
711+
/// Guide".
712+
MachineInstr *buildSDivOrSRemUsingMul(MachineInstr &MI) const;
713+
/// Combine G_SDIV or G_SREM by constant into a multiply by magic constant.
714+
bool matchSDivOrSRemByConst(MachineInstr &MI) const;
715+
void applySDivOrSRemByConst(MachineInstr &MI) const;
715716

716717
/// Given an G_SDIV \p MI expressing a signed divided by a pow2 constant,
717718
/// return expressions that implements it by shifting.

‎llvm/include/llvm/Target/GlobalISel/Combine.td‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1132,14 +1132,14 @@ def form_bitfield_extract : GICombineGroup<[bitfield_extract_from_sext_inreg,
11321132
def udiv_by_const : GICombineRule<
11331133
(defs root:$root),
11341134
(match (G_UDIV $dst, $x, $y):$root,
1135-
[{ return Helper.matchUDivorURemByConst(*${root}); }]),
1136-
(apply [{ Helper.applyUDivorURemByConst(*${root}); }])>;
1135+
[{ return Helper.matchUDivOrURemByConst(*${root}); }]),
1136+
(apply [{ Helper.applyUDivOrURemByConst(*${root}); }])>;
11371137

11381138
def sdiv_by_const : GICombineRule<
11391139
(defs root:$root),
11401140
(match (G_SDIV $dst, $x, $y):$root,
1141-
[{ return Helper.matchSDivByConst(*${root}); }]),
1142-
(apply [{ Helper.applySDivByConst(*${root}); }])>;
1141+
[{ return Helper.matchSDivOrSRemByConst(*${root}); }]),
1142+
(apply [{ Helper.applySDivOrSRemByConst(*${root}); }])>;
11431143

11441144
def sdiv_by_pow2 : GICombineRule<
11451145
(defs root:$root),
@@ -1159,10 +1159,16 @@ def intdiv_combines : GICombineGroup<[udiv_by_pow2, sdiv_by_pow2,
11591159
def urem_by_const : GICombineRule<
11601160
(defs root:$root),
11611161
(match (G_UREM $dst, $x, $y):$root,
1162-
[{ return Helper.matchUDivorURemByConst(*${root}); }]),
1163-
(apply [{ Helper.applyUDivorURemByConst(*${root}); }])>;
1162+
[{ return Helper.matchUDivOrURemByConst(*${root}); }]),
1163+
(apply [{ Helper.applyUDivOrURemByConst(*${root}); }])>;
11641164

1165-
def intrem_combines : GICombineGroup<[urem_by_const]>;
1165+
def srem_by_const : GICombineRule<
1166+
(defs root:$root),
1167+
(match (G_SREM $dst, $x, $y):$root,
1168+
[{ return Helper.matchSDivOrSRemByConst(*${root}); }]),
1169+
(apply [{ Helper.applySDivOrSRemByConst(*${root}); }])>;
1170+
1171+
def intrem_combines : GICombineGroup<[urem_by_const, srem_by_const]>;
11661172

11671173
def reassoc_ptradd : GICombineRule<
11681174
(defs root:$root, build_fn_matchinfo:$matchinfo),

‎llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp‎

Lines changed: 29 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -5300,7 +5300,7 @@ bool CombinerHelper::matchSubAddSameReg(MachineInstr &MI,
53005300
return false;
53015301
}
53025302

5303-
MachineInstr *CombinerHelper::buildUDivorURemUsingMul(MachineInstr &MI) const {
5303+
MachineInstr *CombinerHelper::buildUDivOrURemUsingMul(MachineInstr &MI) const {
53045304
unsigned Opcode = MI.getOpcode();
53055305
assert(Opcode == TargetOpcode::G_UDIV || Opcode == TargetOpcode::G_UREM);
53065306
auto &UDivorRem = cast<GenericMachineInstr>(MI);
@@ -5468,7 +5468,7 @@ MachineInstr *CombinerHelper::buildUDivorURemUsingMul(MachineInstr &MI) const {
54685468
return ret;
54695469
}
54705470

5471-
bool CombinerHelper::matchUDivorURemByConst(MachineInstr &MI) const {
5471+
bool CombinerHelper::matchUDivOrURemByConst(MachineInstr &MI) const {
54725472
unsigned Opcode = MI.getOpcode();
54735473
assert(Opcode == TargetOpcode::G_UDIV || Opcode == TargetOpcode::G_UREM);
54745474
Register Dst = MI.getOperand(0).getReg();
@@ -5517,13 +5517,14 @@ bool CombinerHelper::matchUDivorURemByConst(MachineInstr &MI) const {
55175517
MRI, RHS, [](const Constant *C) { return C && !C->isNullValue(); });
55185518
}
55195519

5520-
void CombinerHelper::applyUDivorURemByConst(MachineInstr &MI) const {
5521-
auto *NewMI = buildUDivorURemUsingMul(MI);
5520+
void CombinerHelper::applyUDivOrURemByConst(MachineInstr &MI) const {
5521+
auto *NewMI = buildUDivOrURemUsingMul(MI);
55225522
replaceSingleDefInstWithReg(MI, NewMI->getOperand(0).getReg());
55235523
}
55245524

5525-
bool CombinerHelper::matchSDivByConst(MachineInstr &MI) const {
5526-
assert(MI.getOpcode() == TargetOpcode::G_SDIV && "Expected SDIV");
5525+
bool CombinerHelper::matchSDivOrSRemByConst(MachineInstr &MI) const {
5526+
unsigned Opcode = MI.getOpcode();
5527+
assert(Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_SREM);
55275528
Register Dst = MI.getOperand(0).getReg();
55285529
Register RHS = MI.getOperand(2).getReg();
55295530
LLT DstTy = MRI.getType(Dst);
@@ -5543,7 +5544,8 @@ bool CombinerHelper::matchSDivByConst(MachineInstr &MI) const {
55435544
return false;
55445545

55455546
// If the sdiv has an 'exact' flag we can use a simpler lowering.
5546-
if (MI.getFlag(MachineInstr::MIFlag::IsExact)) {
5547+
if (Opcode == TargetOpcode::G_SDIV &&
5548+
MI.getFlag(MachineInstr::MIFlag::IsExact)) {
55475549
return matchUnaryPredicate(
55485550
MRI, RHS, [](const Constant *C) { return C && !C->isNullValue(); });
55495551
}
@@ -5559,23 +5561,28 @@ bool CombinerHelper::matchSDivByConst(MachineInstr &MI) const {
55595561
if (!isLegal({TargetOpcode::G_SMULH, {DstTy}}) &&
55605562
!isLegalOrHasWidenScalar({TargetOpcode::G_MUL, {WideTy, WideTy}}))
55615563
return false;
5564+
if (Opcode == TargetOpcode::G_SREM &&
5565+
!isLegalOrBeforeLegalizer({TargetOpcode::G_SUB, {DstTy, DstTy}}))
5566+
return false;
55625567
}
55635568

55645569
return matchUnaryPredicate(
55655570
MRI, RHS, [](const Constant *C) { return C && !C->isNullValue(); });
55665571
}
55675572

5568-
void CombinerHelper::applySDivByConst(MachineInstr &MI) const {
5569-
auto *NewMI = buildSDivUsingMul(MI);
5573+
void CombinerHelper::applySDivOrSRemByConst(MachineInstr &MI) const {
5574+
auto *NewMI = buildSDivOrSRemUsingMul(MI);
55705575
replaceSingleDefInstWithReg(MI, NewMI->getOperand(0).getReg());
55715576
}
55725577

5573-
MachineInstr *CombinerHelper::buildSDivUsingMul(MachineInstr &MI) const {
5574-
assert(MI.getOpcode() == TargetOpcode::G_SDIV && "Expected SDIV");
5575-
auto &SDiv = cast<GenericMachineInstr>(MI);
5576-
Register Dst = SDiv.getReg(0);
5577-
Register LHS = SDiv.getReg(1);
5578-
Register RHS = SDiv.getReg(2);
5578+
MachineInstr *CombinerHelper::buildSDivOrSRemUsingMul(MachineInstr &MI) const {
5579+
unsigned Opcode = MI.getOpcode();
5580+
assert(MI.getOpcode() == TargetOpcode::G_SDIV ||
5581+
Opcode == TargetOpcode::G_SREM);
5582+
auto &SDivorRem = cast<GenericMachineInstr>(MI);
5583+
Register Dst = SDivorRem.getReg(0);
5584+
Register LHS = SDivorRem.getReg(1);
5585+
Register RHS = SDivorRem.getReg(2);
55795586
LLT Ty = MRI.getType(Dst);
55805587
LLT ScalarTy = Ty.getScalarType();
55815588
const unsigned EltBits = ScalarTy.getScalarSizeInBits();
@@ -5705,7 +5712,13 @@ MachineInstr *CombinerHelper::buildSDivUsingMul(MachineInstr &MI) const {
57055712
auto SignShift = MIB.buildConstant(ShiftAmtTy, EltBits - 1);
57065713
auto T = MIB.buildLShr(Ty, Q, SignShift);
57075714
T = MIB.buildAnd(Ty, T, ShiftMask);
5708-
return MIB.buildAdd(Ty, Q, T);
5715+
auto ret = MIB.buildAdd(Ty, Q, T);
5716+
5717+
if (Opcode == TargetOpcode::G_SREM) {
5718+
auto Prod = MIB.buildMul(Ty, ret, RHS);
5719+
return MIB.buildSub(Ty, LHS, Prod);
5720+
}
5721+
return ret;
57095722
}
57105723

57115724
bool CombinerHelper::matchDivByPow2(MachineInstr &MI, bool IsSigned) const {

0 commit comments

Comments
 (0)