Skip to content

Commit 9b692e5

Browse files
author
Thorsten Schütt
authored
[GlobalIsel] Combine G_VSCALE (#94096)
We need them for scalable address calculation and legal scalable addressing modes.
1 parent 3cdb1df commit 9b692e5

File tree

5 files changed

+282
-3
lines changed

5 files changed

+282
-3
lines changed

‎llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -871,6 +871,14 @@ class CombinerHelper {
871871
bool matchFreezeOfSingleMaybePoisonOperand(MachineInstr &MI,
872872
BuildFnTy &MatchInfo);
873873

874+
bool matchAddOfVScale(const MachineOperand &MO, BuildFnTy &MatchInfo);
875+
876+
bool matchMulOfVScale(const MachineOperand &MO, BuildFnTy &MatchInfo);
877+
878+
bool matchSubOfVScale(const MachineOperand &MO, BuildFnTy &MatchInfo);
879+
880+
bool matchShlOfVScale(const MachineOperand &MO, BuildFnTy &MatchInfo);
881+
874882
private:
875883
/// Checks for legality of an indexed variant of \p LdSt.
876884
bool isIndexedLoadStoreLegal(GLoadStore &LdSt) const;

‎llvm/include/llvm/CodeGen/GlobalISel/GenericMachineInstrs.h‎

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,12 @@
1414
#ifndef LLVM_CODEGEN_GLOBALISEL_GENERICMACHINEINSTRS_H
1515
#define LLVM_CODEGEN_GLOBALISEL_GENERICMACHINEINSTRS_H
1616

17-
#include "llvm/IR/Instructions.h"
17+
#include "llvm/ADT/APInt.h"
1818
#include "llvm/CodeGen/MachineInstr.h"
1919
#include "llvm/CodeGen/MachineMemOperand.h"
2020
#include "llvm/CodeGen/TargetOpcodes.h"
21+
#include "llvm/IR/Constants.h"
22+
#include "llvm/IR/Instructions.h"
2123
#include "llvm/Support/Casting.h"
2224

2325
namespace llvm {
@@ -856,6 +858,43 @@ class GTrunc : public GCastOp {
856858
};
857859
};
858860

861+
/// Represents a vscale.
862+
class GVScale : public GenericMachineInstr {
863+
public:
864+
APInt getSrc() const { return getOperand(1).getCImm()->getValue(); }
865+
866+
static bool classof(const MachineInstr *MI) {
867+
return MI->getOpcode() == TargetOpcode::G_VSCALE;
868+
};
869+
};
870+
871+
/// Represents an integer subtraction.
872+
class GSub : public GIntBinOp {
873+
public:
874+
static bool classof(const MachineInstr *MI) {
875+
return MI->getOpcode() == TargetOpcode::G_SUB;
876+
};
877+
};
878+
879+
/// Represents an integer multiplication.
880+
class GMul : public GIntBinOp {
881+
public:
882+
static bool classof(const MachineInstr *MI) {
883+
return MI->getOpcode() == TargetOpcode::G_MUL;
884+
};
885+
};
886+
887+
/// Represents a shift left.
888+
class GShl : public GenericMachineInstr {
889+
public:
890+
Register getSrcReg() const { return getOperand(1).getReg(); }
891+
Register getShiftReg() const { return getOperand(2).getReg(); }
892+
893+
static bool classof(const MachineInstr *MI) {
894+
return MI->getOpcode() == TargetOpcode::G_SHL;
895+
};
896+
};
897+
859898
} // namespace llvm
860899

861900
#endif // LLVM_CODEGEN_GLOBALISEL_GENERICMACHINEINSTRS_H

‎llvm/include/llvm/Target/GlobalISel/Combine.td‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1605,6 +1605,37 @@ def insert_vector_elt_oob : GICombineRule<
16051605
[{ return Helper.matchInsertVectorElementOOB(*${root}, ${matchinfo}); }]),
16061606
(apply [{ Helper.applyBuildFn(*${root}, ${matchinfo}); }])>;
16071607

1608+
def add_of_vscale : GICombineRule<
1609+
(defs root:$root, build_fn_matchinfo:$matchinfo),
1610+
(match (G_VSCALE $left, $imm1),
1611+
(G_VSCALE $right, $imm2),
1612+
(G_ADD $root, $left, $right, (MIFlags NoSWrap)),
1613+
[{ return Helper.matchAddOfVScale(${root}, ${matchinfo}); }]),
1614+
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1615+
1616+
def mul_of_vscale : GICombineRule<
1617+
(defs root:$root, build_fn_matchinfo:$matchinfo),
1618+
(match (G_VSCALE $left, $scale),
1619+
(G_CONSTANT $x, $imm1),
1620+
(G_MUL $root, $left, $x, (MIFlags NoSWrap)),
1621+
[{ return Helper.matchMulOfVScale(${root}, ${matchinfo}); }]),
1622+
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1623+
1624+
def shl_of_vscale : GICombineRule<
1625+
(defs root:$root, build_fn_matchinfo:$matchinfo),
1626+
(match (G_VSCALE $left, $imm),
1627+
(G_CONSTANT $x, $imm1),
1628+
(G_SHL $root, $left, $x, (MIFlags NoSWrap)),
1629+
[{ return Helper.matchShlOfVScale(${root}, ${matchinfo}); }]),
1630+
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1631+
1632+
def sub_of_vscale : GICombineRule<
1633+
(defs root:$root, build_fn_matchinfo:$matchinfo),
1634+
(match (G_VSCALE $right, $imm),
1635+
(G_SUB $root, $x, $right, (MIFlags NoSWrap)),
1636+
[{ return Helper.matchSubOfVScale(${root}, ${matchinfo}); }]),
1637+
(apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
1638+
16081639
// match_extract_of_element and insert_vector_elt_oob must be the first!
16091640
def vector_ops_combines: GICombineGroup<[
16101641
match_extract_of_element_undef_vector,
@@ -1637,7 +1668,11 @@ extract_vector_element_build_vector_trunc6,
16371668
extract_vector_element_build_vector_trunc7,
16381669
extract_vector_element_build_vector_trunc8,
16391670
extract_vector_element_shuffle_vector,
1640-
insert_vector_element_extract_vector_element
1671+
insert_vector_element_extract_vector_element,
1672+
add_of_vscale,
1673+
mul_of_vscale,
1674+
shl_of_vscale,
1675+
sub_of_vscale,
16411676
]>;
16421677

16431678

‎llvm/lib/CodeGen/GlobalISel/CombinerHelperVectorOps.cpp‎

Lines changed: 85 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@
66
//
77
//===----------------------------------------------------------------------===//
88
//
9-
// This file implements CombinerHelper for G_EXTRACT_VECTOR_ELT.
9+
// This file implements CombinerHelper for G_EXTRACT_VECTOR_ELT,
10+
// G_INSERT_VECTOR_ELT, and G_VSCALE
1011
//
1112
//===----------------------------------------------------------------------===//
1213
#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
@@ -400,3 +401,86 @@ bool CombinerHelper::matchInsertVectorElementOOB(MachineInstr &MI,
400401

401402
return false;
402403
}
404+
405+
bool CombinerHelper::matchAddOfVScale(const MachineOperand &MO,
406+
BuildFnTy &MatchInfo) {
407+
GAdd *Add = cast<GAdd>(MRI.getVRegDef(MO.getReg()));
408+
GVScale *LHSVScale = cast<GVScale>(MRI.getVRegDef(Add->getLHSReg()));
409+
GVScale *RHSVScale = cast<GVScale>(MRI.getVRegDef(Add->getRHSReg()));
410+
411+
Register Dst = Add->getReg(0);
412+
413+
if (!MRI.hasOneNonDBGUse(LHSVScale->getReg(0)) ||
414+
!MRI.hasOneNonDBGUse(RHSVScale->getReg(0)))
415+
return false;
416+
417+
MatchInfo = [=](MachineIRBuilder &B) {
418+
B.buildVScale(Dst, LHSVScale->getSrc() + RHSVScale->getSrc());
419+
};
420+
421+
return true;
422+
}
423+
424+
bool CombinerHelper::matchMulOfVScale(const MachineOperand &MO,
425+
BuildFnTy &MatchInfo) {
426+
GMul *Mul = cast<GMul>(MRI.getVRegDef(MO.getReg()));
427+
GVScale *LHSVScale = cast<GVScale>(MRI.getVRegDef(Mul->getLHSReg()));
428+
429+
std::optional<APInt> MaybeRHS = getIConstantVRegVal(Mul->getRHSReg(), MRI);
430+
if (!MaybeRHS)
431+
return false;
432+
433+
Register Dst = MO.getReg();
434+
435+
if (!MRI.hasOneNonDBGUse(LHSVScale->getReg(0)))
436+
return false;
437+
438+
MatchInfo = [=](MachineIRBuilder &B) {
439+
B.buildVScale(Dst, LHSVScale->getSrc() * *MaybeRHS);
440+
};
441+
442+
return true;
443+
}
444+
445+
bool CombinerHelper::matchSubOfVScale(const MachineOperand &MO,
446+
BuildFnTy &MatchInfo) {
447+
GSub *Sub = cast<GSub>(MRI.getVRegDef(MO.getReg()));
448+
GVScale *RHSVScale = cast<GVScale>(MRI.getVRegDef(Sub->getRHSReg()));
449+
450+
Register Dst = MO.getReg();
451+
LLT DstTy = MRI.getType(Dst);
452+
453+
if (!MRI.hasOneNonDBGUse(RHSVScale->getReg(0)) ||
454+
!isLegalOrBeforeLegalizer({TargetOpcode::G_ADD, DstTy}))
455+
return false;
456+
457+
MatchInfo = [=](MachineIRBuilder &B) {
458+
auto VScale = B.buildVScale(DstTy, -RHSVScale->getSrc());
459+
B.buildAdd(Dst, Sub->getLHSReg(), VScale, Sub->getFlags());
460+
};
461+
462+
return true;
463+
}
464+
465+
bool CombinerHelper::matchShlOfVScale(const MachineOperand &MO,
466+
BuildFnTy &MatchInfo) {
467+
GShl *Shl = cast<GShl>(MRI.getVRegDef(MO.getReg()));
468+
GVScale *LHSVScale = cast<GVScale>(MRI.getVRegDef(Shl->getSrcReg()));
469+
470+
std::optional<APInt> MaybeRHS = getIConstantVRegVal(Shl->getShiftReg(), MRI);
471+
if (!MaybeRHS)
472+
return false;
473+
474+
Register Dst = MO.getReg();
475+
LLT DstTy = MRI.getType(Dst);
476+
477+
if (!MRI.hasOneNonDBGUse(LHSVScale->getReg(0)) ||
478+
!isLegalOrBeforeLegalizer({TargetOpcode::G_VSCALE, DstTy}))
479+
return false;
480+
481+
MatchInfo = [=](MachineIRBuilder &B) {
482+
B.buildVScale(Dst, LHSVScale->getSrc().shl(*MaybeRHS));
483+
};
484+
485+
return true;
486+
}

‎llvm/test/CodeGen/AArch64/GlobalISel/combine-vscale.mir‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
2+
# RUN: llc -o - -mtriple=aarch64 -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s | FileCheck %s
3+
4+
...
5+
---
6+
name: sum_of_vscale
7+
body: |
8+
bb.1:
9+
liveins: $x0, $x1
10+
; CHECK-LABEL: name: sum_of_vscale
11+
; CHECK: liveins: $x0, $x1
12+
; CHECK-NEXT: {{ $}}
13+
; CHECK-NEXT: %sum:_(s64) = G_VSCALE i64 20
14+
; CHECK-NEXT: $x0 = COPY %sum(s64)
15+
; CHECK-NEXT: RET_ReallyLR implicit $x0
16+
%rhs:_(s64) = G_VSCALE i64 11
17+
%lhs:_(s64) = G_VSCALE i64 9
18+
%sum:_(s64) = nsw G_ADD %lhs(s64), %rhs(s64)
19+
$x0 = COPY %sum(s64)
20+
RET_ReallyLR implicit $x0
21+
...
22+
---
23+
name: sum_of_vscale_multi_use
24+
body: |
25+
bb.1:
26+
liveins: $x0, $x1
27+
; CHECK-LABEL: name: sum_of_vscale_multi_use
28+
; CHECK: liveins: $x0, $x1
29+
; CHECK-NEXT: {{ $}}
30+
; CHECK-NEXT: %rhs:_(s64) = G_VSCALE i64 11
31+
; CHECK-NEXT: %lhs:_(s64) = G_VSCALE i64 9
32+
; CHECK-NEXT: %sum:_(s64) = nsw G_ADD %lhs, %rhs
33+
; CHECK-NEXT: $x0 = COPY %sum(s64)
34+
; CHECK-NEXT: $x1 = COPY %rhs(s64)
35+
; CHECK-NEXT: RET_ReallyLR implicit $x0
36+
%rhs:_(s64) = G_VSCALE i64 11
37+
%lhs:_(s64) = G_VSCALE i64 9
38+
%sum:_(s64) = nsw G_ADD %lhs(s64), %rhs(s64)
39+
$x0 = COPY %sum(s64)
40+
$x1 = COPY %rhs(s64)
41+
RET_ReallyLR implicit $x0
42+
...
43+
---
44+
name: mul_of_vscale
45+
body: |
46+
bb.1:
47+
liveins: $x0, $x1
48+
; CHECK-LABEL: name: mul_of_vscale
49+
; CHECK: liveins: $x0, $x1
50+
; CHECK-NEXT: {{ $}}
51+
; CHECK-NEXT: %mul:_(s64) = G_VSCALE i64 99
52+
; CHECK-NEXT: $x0 = COPY %mul(s64)
53+
; CHECK-NEXT: RET_ReallyLR implicit $x0
54+
%rhs:_(s64) = G_CONSTANT i64 11
55+
%lhs:_(s64) = G_VSCALE i64 9
56+
%mul:_(s64) = nsw G_MUL %lhs(s64), %rhs(s64)
57+
$x0 = COPY %mul(s64)
58+
RET_ReallyLR implicit $x0
59+
...
60+
---
61+
name: sub_of_vscale
62+
body: |
63+
bb.1:
64+
liveins: $x0, $x1
65+
; CHECK-LABEL: name: sub_of_vscale
66+
; CHECK: liveins: $x0, $x1
67+
; CHECK-NEXT: {{ $}}
68+
; CHECK-NEXT: %x:_(s64) = COPY $x0
69+
; CHECK-NEXT: [[VSCALE:%[0-9]+]]:_(s64) = G_VSCALE i64 -9
70+
; CHECK-NEXT: %sub:_(s64) = nsw G_ADD %x, [[VSCALE]]
71+
; CHECK-NEXT: $x0 = COPY %sub(s64)
72+
; CHECK-NEXT: RET_ReallyLR implicit $x0
73+
%x:_(s64) = COPY $x0
74+
%rhs:_(s64) = G_VSCALE i64 9
75+
%sub:_(s64) = nsw G_SUB %x(s64), %rhs(s64)
76+
$x0 = COPY %sub(s64)
77+
RET_ReallyLR implicit $x0
78+
...
79+
---
80+
name: shl_of_vscale
81+
body: |
82+
bb.1:
83+
liveins: $x0, $x1
84+
; CHECK-LABEL: name: shl_of_vscale
85+
; CHECK: liveins: $x0, $x1
86+
; CHECK-NEXT: {{ $}}
87+
; CHECK-NEXT: %shl:_(s64) = G_VSCALE i64 44
88+
; CHECK-NEXT: $x0 = COPY %shl(s64)
89+
; CHECK-NEXT: RET_ReallyLR implicit $x0
90+
%rhs:_(s64) = G_CONSTANT i64 2
91+
%lhs:_(s64) = G_VSCALE i64 11
92+
%shl:_(s64) = nsw G_SHL %lhs(s64), %rhs(s64)
93+
$x0 = COPY %shl(s64)
94+
RET_ReallyLR implicit $x0
95+
...
96+
---
97+
name: shl_of_vscale_wrong_flag
98+
body: |
99+
bb.1:
100+
liveins: $x0, $x1
101+
; CHECK-LABEL: name: shl_of_vscale_wrong_flag
102+
; CHECK: liveins: $x0, $x1
103+
; CHECK-NEXT: {{ $}}
104+
; CHECK-NEXT: %rhs:_(s64) = G_CONSTANT i64 2
105+
; CHECK-NEXT: %lhs:_(s64) = G_VSCALE i64 11
106+
; CHECK-NEXT: %shl:_(s64) = nuw G_SHL %lhs, %rhs(s64)
107+
; CHECK-NEXT: $x0 = COPY %shl(s64)
108+
; CHECK-NEXT: RET_ReallyLR implicit $x0
109+
%rhs:_(s64) = G_CONSTANT i64 2
110+
%lhs:_(s64) = G_VSCALE i64 11
111+
%shl:_(s64) = nuw G_SHL %lhs(s64), %rhs(s64)
112+
$x0 = COPY %shl(s64)
113+
RET_ReallyLR implicit $x0

0 commit comments

Comments
 (0)