Skip to content

Commit b733229

Browse files
david-armdavid-arm
committed
[DAGCombiner] Add support for scalarising extracts of a vector setcc
For IR like this: %icmp = icmp ult <4 x i32> %a, splat (i32 5) %res = extractelement <4 x i1> %icmp, i32 1 where there is only one use of %icmp we can take a similar approach to what we already do for binary ops such add, sub, etc. and convert this into %ext = extractelement <4 x i32> %a, i32 1 %res = icmp ult i32 %ext, 5 For AArch64 targets at least the scalar boolean result will almost certainly need to be in a GPR anyway, since it will probably be used by branches for control flow. I've tried to reuse existing code in scalarizeExtractedBinop to also work for setcc. NOTE: The optimisations don't apply for tests such as extract_icmp_v4i32_splat_rhs in the file CodeGen/AArch64/extract-vector-cmp.ll because scalarizeExtractedBinOp only works if one of the input operands is a constant.
1 parent 4717e0c commit b733229

File tree

6 files changed

+61
-61
lines changed

6 files changed

+61
-61
lines changed

‎llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp‎

Lines changed: 26 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -22749,16 +22749,21 @@ SDValue DAGCombiner::scalarizeExtractedVectorLoad(SDNode *EVE, EVT InVecVT,
2274922749

2275022750
/// Transform a vector binary operation into a scalar binary operation by moving
2275122751
/// the math/logic after an extract element of a vector.
22752-
static SDValue scalarizeExtractedBinop(SDNode *ExtElt, SelectionDAG &DAG,
22753-
const SDLoc &DL, bool LegalOperations) {
22752+
static SDValue scalarizeExtractedBinOp(SDNode *ExtElt, SelectionDAG &DAG,
22753+
const SDLoc &DL) {
2275422754
const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2275522755
SDValue Vec = ExtElt->getOperand(0);
2275622756
SDValue Index = ExtElt->getOperand(1);
2275722757
auto *IndexC = dyn_cast<ConstantSDNode>(Index);
22758-
if (!IndexC || !TLI.isBinOp(Vec.getOpcode()) || !Vec.hasOneUse() ||
22758+
unsigned Opc = Vec.getOpcode();
22759+
if (!IndexC || !Vec.hasOneUse() || (!TLI.isBinOp(Opc) && Opc != ISD::SETCC) ||
2275922760
Vec->getNumValues() != 1)
2276022761
return SDValue();
2276122762

22763+
EVT ResVT = ExtElt->getValueType(0);
22764+
if (Opc == ISD::SETCC && ResVT != Vec.getValueType().getVectorElementType())
22765+
return SDValue();
22766+
2276222767
// Targets may want to avoid this to prevent an expensive register transfer.
2276322768
if (!TLI.shouldScalarizeBinop(Vec))
2276422769
return SDValue();
@@ -22769,19 +22774,24 @@ static SDValue scalarizeExtractedBinop(SDNode *ExtElt, SelectionDAG &DAG,
2276922774
SDValue Op0 = Vec.getOperand(0);
2277022775
SDValue Op1 = Vec.getOperand(1);
2277122776
APInt SplatVal;
22772-
if (isAnyConstantBuildVector(Op0, true) ||
22773-
ISD::isConstantSplatVector(Op0.getNode(), SplatVal) ||
22774-
isAnyConstantBuildVector(Op1, true) ||
22775-
ISD::isConstantSplatVector(Op1.getNode(), SplatVal)) {
22776-
// extractelt (binop X, C), IndexC --> binop (extractelt X, IndexC), C'
22777-
// extractelt (binop C, X), IndexC --> binop C', (extractelt X, IndexC)
22778-
EVT VT = ExtElt->getValueType(0);
22779-
SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op0, Index);
22780-
SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op1, Index);
22781-
return DAG.getNode(Vec.getOpcode(), DL, VT, Ext0, Ext1);
22782-
}
22777+
if (!isAnyConstantBuildVector(Op0, true) &&
22778+
!ISD::isConstantSplatVector(Op0.getNode(), SplatVal) &&
22779+
!isAnyConstantBuildVector(Op1, true) &&
22780+
!ISD::isConstantSplatVector(Op1.getNode(), SplatVal))
22781+
return SDValue();
2278322782

22784-
return SDValue();
22783+
// extractelt (op X, C), IndexC --> op (extractelt X, IndexC), C'
22784+
// extractelt (op C, X), IndexC --> op C', (extractelt X, IndexC)
22785+
if (Opc == ISD::SETCC) {
22786+
EVT OpVT = Op0->getValueType(0).getVectorElementType();
22787+
Op0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT, Op0, Index);
22788+
Op1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT, Op1, Index);
22789+
return DAG.getSetCC(DL, ResVT, Op0, Op1,
22790+
cast<CondCodeSDNode>(Vec->getOperand(2))->get());
22791+
}
22792+
Op0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Op0, Index);
22793+
Op1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResVT, Op1, Index);
22794+
return DAG.getNode(Opc, DL, ResVT, Op0, Op1);
2278522795
}
2278622796

2278722797
// Given a ISD::EXTRACT_VECTOR_ELT, which is a glorified bit sequence extract,
@@ -23014,7 +23024,7 @@ SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) {
2301423024
}
2301523025
}
2301623026

23017-
if (SDValue BO = scalarizeExtractedBinop(N, DAG, DL, LegalOperations))
23027+
if (SDValue BO = scalarizeExtractedBinOp(N, DAG, DL))
2301823028
return BO;
2301923029

2302023030
if (VecVT.isScalableVector())

‎llvm/lib/Target/AArch64/AArch64ISelLowering.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1348,6 +1348,10 @@ class AArch64TargetLowering : public TargetLowering {
13481348
unsigned getMinimumJumpTableEntries() const override;
13491349

13501350
bool softPromoteHalfType() const override { return true; }
1351+
1352+
bool shouldScalarizeBinop(SDValue VecOp) const override {
1353+
return VecOp.getOpcode() == ISD::SETCC;
1354+
}
13511355
};
13521356

13531357
namespace AArch64 {

‎llvm/lib/Target/RISCV/RISCVISelLowering.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2093,7 +2093,7 @@ bool RISCVTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
20932093

20942094
// Assume target opcodes can't be scalarized.
20952095
// TODO - do we have any exceptions?
2096-
if (Opc >= ISD::BUILTIN_OP_END)
2096+
if (Opc >= ISD::BUILTIN_OP_END || !isBinOp(Opc))
20972097
return false;
20982098

20992099
// If the vector op is not supported, try to convert to scalar.

‎llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -429,7 +429,7 @@ bool WebAssemblyTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
429429

430430
// Assume target opcodes can't be scalarized.
431431
// TODO - do we have any exceptions?
432-
if (Opc >= ISD::BUILTIN_OP_END)
432+
if (Opc >= ISD::BUILTIN_OP_END || !isBinOp(Opc))
433433
return false;
434434

435435
// If the vector op is not supported, try to convert to scalar.

‎llvm/lib/Target/X86/X86ISelLowering.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3306,7 +3306,7 @@ bool X86TargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
33063306

33073307
// Assume target opcodes can't be scalarized.
33083308
// TODO - do we have any exceptions?
3309-
if (Opc >= ISD::BUILTIN_OP_END)
3309+
if (Opc >= ISD::BUILTIN_OP_END || !isBinOp(Opc))
33103310
return false;
33113311

33123312
// If the vector op is not supported, try to convert to scalar.

‎llvm/test/CodeGen/AArch64/extract-vector-cmp.ll‎

Lines changed: 28 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,9 @@ target triple = "aarch64-unknown-linux-gnu"
77
define i1 @extract_icmp_v4i32_const_splat_rhs(<4 x i32> %a) {
88
; CHECK-LABEL: extract_icmp_v4i32_const_splat_rhs:
99
; CHECK: // %bb.0:
10-
; CHECK-NEXT: movi v1.4s, #5
11-
; CHECK-NEXT: cmhi v0.4s, v1.4s, v0.4s
12-
; CHECK-NEXT: xtn v0.4h, v0.4s
13-
; CHECK-NEXT: umov w8, v0.h[1]
14-
; CHECK-NEXT: and w0, w8, #0x1
10+
; CHECK-NEXT: mov w8, v0.s[1]
11+
; CHECK-NEXT: cmp w8, #5
12+
; CHECK-NEXT: cset w0, lo
1513
; CHECK-NEXT: ret
1614
%icmp = icmp ult <4 x i32> %a, splat (i32 5)
1715
%ext = extractelement <4 x i1> %icmp, i32 1
@@ -21,11 +19,9 @@ define i1 @extract_icmp_v4i32_const_splat_rhs(<4 x i32> %a) {
2119
define i1 @extract_icmp_v4i32_const_splat_lhs(<4 x i32> %a) {
2220
; CHECK-LABEL: extract_icmp_v4i32_const_splat_lhs:
2321
; CHECK: // %bb.0:
24-
; CHECK-NEXT: movi v1.4s, #7
25-
; CHECK-NEXT: cmhi v0.4s, v0.4s, v1.4s
26-
; CHECK-NEXT: xtn v0.4h, v0.4s
27-
; CHECK-NEXT: umov w8, v0.h[1]
28-
; CHECK-NEXT: and w0, w8, #0x1
22+
; CHECK-NEXT: mov w8, v0.s[1]
23+
; CHECK-NEXT: cmp w8, #7
24+
; CHECK-NEXT: cset w0, hi
2925
; CHECK-NEXT: ret
3026
%icmp = icmp ult <4 x i32> splat(i32 7), %a
3127
%ext = extractelement <4 x i1> %icmp, i32 1
@@ -35,12 +31,9 @@ define i1 @extract_icmp_v4i32_const_splat_lhs(<4 x i32> %a) {
3531
define i1 @extract_icmp_v4i32_const_vec_rhs(<4 x i32> %a) {
3632
; CHECK-LABEL: extract_icmp_v4i32_const_vec_rhs:
3733
; CHECK: // %bb.0:
38-
; CHECK-NEXT: adrp x8, .LCPI2_0
39-
; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI2_0]
40-
; CHECK-NEXT: cmhi v0.4s, v1.4s, v0.4s
41-
; CHECK-NEXT: xtn v0.4h, v0.4s
42-
; CHECK-NEXT: umov w8, v0.h[1]
43-
; CHECK-NEXT: and w0, w8, #0x1
34+
; CHECK-NEXT: mov w8, v0.s[1]
35+
; CHECK-NEXT: cmp w8, #234
36+
; CHECK-NEXT: cset w0, lo
4437
; CHECK-NEXT: ret
4538
%icmp = icmp ult <4 x i32> %a, <i32 5, i32 234, i32 -1, i32 7>
4639
%ext = extractelement <4 x i1> %icmp, i32 1
@@ -50,12 +43,10 @@ define i1 @extract_icmp_v4i32_const_vec_rhs(<4 x i32> %a) {
5043
define i1 @extract_fcmp_v4f32_const_splat_rhs(<4 x float> %a) {
5144
; CHECK-LABEL: extract_fcmp_v4f32_const_splat_rhs:
5245
; CHECK: // %bb.0:
53-
; CHECK-NEXT: fmov v1.4s, #4.00000000
54-
; CHECK-NEXT: fcmge v0.4s, v0.4s, v1.4s
55-
; CHECK-NEXT: mvn v0.16b, v0.16b
56-
; CHECK-NEXT: xtn v0.4h, v0.4s
57-
; CHECK-NEXT: umov w8, v0.h[1]
58-
; CHECK-NEXT: and w0, w8, #0x1
46+
; CHECK-NEXT: mov s0, v0.s[1]
47+
; CHECK-NEXT: fmov s1, #4.00000000
48+
; CHECK-NEXT: fcmp s0, s1
49+
; CHECK-NEXT: cset w0, lt
5950
; CHECK-NEXT: ret
6051
%fcmp = fcmp ult <4 x float> %a, splat(float 4.0e+0)
6152
%ext = extractelement <4 x i1> %fcmp, i32 1
@@ -66,39 +57,34 @@ define void @vector_loop_with_icmp(ptr nocapture noundef writeonly %dest) {
6657
; CHECK-LABEL: vector_loop_with_icmp:
6758
; CHECK: // %bb.0: // %entry
6859
; CHECK-NEXT: index z0.d, #0, #1
69-
; CHECK-NEXT: mov w8, #15 // =0xf
70-
; CHECK-NEXT: mov w9, #2 // =0x2
60+
; CHECK-NEXT: mov w8, #2 // =0x2
61+
; CHECK-NEXT: mov w9, #16 // =0x10
7162
; CHECK-NEXT: dup v1.2d, x8
72-
; CHECK-NEXT: dup v2.2d, x9
73-
; CHECK-NEXT: add x9, x0, #4
74-
; CHECK-NEXT: mov w10, #16 // =0x10
75-
; CHECK-NEXT: mov w11, #1 // =0x1
63+
; CHECK-NEXT: add x8, x0, #4
64+
; CHECK-NEXT: mov w10, #1 // =0x1
7665
; CHECK-NEXT: b .LBB4_2
7766
; CHECK-NEXT: .LBB4_1: // %pred.store.continue6
7867
; CHECK-NEXT: // in Loop: Header=BB4_2 Depth=1
79-
; CHECK-NEXT: add v0.2d, v0.2d, v2.2d
80-
; CHECK-NEXT: subs x10, x10, #2
81-
; CHECK-NEXT: add x9, x9, #8
68+
; CHECK-NEXT: add v0.2d, v0.2d, v1.2d
69+
; CHECK-NEXT: subs x9, x9, #2
70+
; CHECK-NEXT: add x8, x8, #8
8271
; CHECK-NEXT: b.eq .LBB4_6
8372
; CHECK-NEXT: .LBB4_2: // %vector.body
8473
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
85-
; CHECK-NEXT: cmhi v3.2d, v1.2d, v0.2d
86-
; CHECK-NEXT: xtn v3.2s, v3.2d
87-
; CHECK-NEXT: fmov w12, s3
88-
; CHECK-NEXT: tbz w12, #0, .LBB4_4
74+
; CHECK-NEXT: fmov x11, d0
75+
; CHECK-NEXT: cmp x11, #14
76+
; CHECK-NEXT: b.hi .LBB4_4
8977
; CHECK-NEXT: // %bb.3: // %pred.store.if
9078
; CHECK-NEXT: // in Loop: Header=BB4_2 Depth=1
91-
; CHECK-NEXT: stur w11, [x9, #-4]
79+
; CHECK-NEXT: stur w10, [x8, #-4]
9280
; CHECK-NEXT: .LBB4_4: // %pred.store.continue
9381
; CHECK-NEXT: // in Loop: Header=BB4_2 Depth=1
94-
; CHECK-NEXT: dup v3.2d, x8
95-
; CHECK-NEXT: cmhi v3.2d, v3.2d, v0.2d
96-
; CHECK-NEXT: xtn v3.2s, v3.2d
97-
; CHECK-NEXT: mov w12, v3.s[1]
98-
; CHECK-NEXT: tbz w12, #0, .LBB4_1
82+
; CHECK-NEXT: mov x11, v0.d[1]
83+
; CHECK-NEXT: cmp x11, #14
84+
; CHECK-NEXT: b.hi .LBB4_1
9985
; CHECK-NEXT: // %bb.5: // %pred.store.if5
10086
; CHECK-NEXT: // in Loop: Header=BB4_2 Depth=1
101-
; CHECK-NEXT: str w11, [x9]
87+
; CHECK-NEXT: str w10, [x8]
10288
; CHECK-NEXT: b .LBB4_1
10389
; CHECK-NEXT: .LBB4_6: // %for.cond.cleanup
10490
; CHECK-NEXT: ret

0 commit comments

Comments
 (0)