Skip to content

Commit 8bb10cf

Browse files
guy-davidguy-david
committed
[AArch64] Optimize DUP of extending loads to avoid GPR->FPR transfer
Loads the data into the SIMD register, thus sparing a physical register and a potentially costly movement of data.
1 parent 6345222 commit 8bb10cf

File tree

5 files changed

+195
-15
lines changed

5 files changed

+195
-15
lines changed

‎llvm/lib/Target/AArch64/AArch64ISelLowering.cpp‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26632,11 +26632,34 @@ static SDValue performDUPCombine(SDNode *N,
2663226632
}
2663326633

2663426634
if (N->getOpcode() == AArch64ISD::DUP) {
26635+
SDValue Op = N->getOperand(0);
26636+
26637+
// Optimize DUP(extload/zextload i8/i16) to avoid GPR->FPR transfer.
26638+
// For example:
26639+
// v4i32 = DUP (i32 (zextloadi8 addr))
26640+
// =>
26641+
// v4i32 = SCALAR_TO_VECTOR (i32 (zextloadi8 addr)) ; Matches to ldr b0
26642+
// v4i32 = DUPLANE32 (v4i32), 0
26643+
if (auto *LD = dyn_cast<LoadSDNode>(Op)) {
26644+
ISD::LoadExtType ExtType = LD->getExtensionType();
26645+
EVT MemVT = LD->getMemoryVT();
26646+
EVT ElemVT = VT.getVectorElementType();
26647+
if ((ExtType == ISD::EXTLOAD || ExtType == ISD::ZEXTLOAD) &&
26648+
(MemVT == MVT::i8 || MemVT == MVT::i16) && ElemVT != MemVT &&
26649+
LD->hasOneUse()) {
26650+
EVT Vec128VT = EVT::getVectorVT(*DCI.DAG.getContext(), ElemVT,
26651+
128 / ElemVT.getSizeInBits());
26652+
SDValue ScalarToVec =
26653+
DCI.DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, Vec128VT, Op);
26654+
return DCI.DAG.getNode(getDUPLANEOp(ElemVT), DL, VT, ScalarToVec,
26655+
DCI.DAG.getConstant(0, DL, MVT::i64));
26656+
}
26657+
}
26658+
2663526659
// If the instruction is known to produce a scalar in SIMD registers, we can
2663626660
// duplicate it across the vector lanes using DUPLANE instead of moving it
2663726661
// to a GPR first. For example, this allows us to handle:
2663826662
// v4i32 = DUP (i32 (FCMGT (f32, f32)))
26639-
SDValue Op = N->getOperand(0);
2664026663
// FIXME: Ideally, we should be able to handle all instructions that
2664126664
// produce a scalar value in FPRs.
2664226665
if (Op.getOpcode() == AArch64ISD::FCMEQ ||

‎llvm/lib/Target/AArch64/AArch64InstrInfo.td‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4375,6 +4375,26 @@ def : Pat <(v1i64 (scalar_to_vector (i64
43754375
(load (ro64.Xpat GPR64sp:$Rn, GPR64:$Rm, ro64.Xext:$extend))))),
43764376
(LDRDroX GPR64sp:$Rn, GPR64:$Rm, ro64.Xext:$extend)>;
43774377

4378+
// Patterns for scalar_to_vector with zero-extended loads.
4379+
// Enables direct SIMD register loads for small integer types (i8/i16) that are
4380+
// naturally zero-extended to i32/i64.
4381+
multiclass ScalarToVectorExtLoad<ValueType VecTy, ValueType ScalarTy> {
4382+
def : Pat<(VecTy (scalar_to_vector (ScalarTy (zextloadi8 (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset))))),
4383+
(SUBREG_TO_REG (i64 0), (LDRBui GPR64sp:$Rn, uimm12s1:$offset), bsub)>;
4384+
def : Pat<(VecTy (scalar_to_vector (ScalarTy (extloadi8 (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset))))),
4385+
(SUBREG_TO_REG (i64 0), (LDRBui GPR64sp:$Rn, uimm12s1:$offset), bsub)>;
4386+
4387+
def : Pat<(VecTy (scalar_to_vector (ScalarTy (zextloadi16 (am_indexed16 GPR64sp:$Rn, uimm12s2:$offset))))),
4388+
(SUBREG_TO_REG (i64 0), (LDRHui GPR64sp:$Rn, uimm12s2:$offset), hsub)>;
4389+
def : Pat<(VecTy (scalar_to_vector (ScalarTy (extloadi16 (am_indexed16 GPR64sp:$Rn, uimm12s2:$offset))))),
4390+
(SUBREG_TO_REG (i64 0), (LDRHui GPR64sp:$Rn, uimm12s2:$offset), hsub)>;
4391+
}
4392+
4393+
defm : ScalarToVectorExtLoad<v16i8, i32>;
4394+
defm : ScalarToVectorExtLoad<v8i16, i32>;
4395+
defm : ScalarToVectorExtLoad<v4i32, i32>;
4396+
defm : ScalarToVectorExtLoad<v2i64, i64>;
4397+
43784398
// Pre-fetch.
43794399
defm PRFUM : PrefetchUnscaled<0b11, 0, 0b10, "prfum",
43804400
[(AArch64Prefetch timm:$Rt,

‎llvm/test/CodeGen/AArch64/aarch64-smull.ll‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -222,22 +222,20 @@ define <4 x i32> @smull_zext_v4i16_v4i32(ptr %A, ptr %B) nounwind {
222222
define <2 x i64> @smull_zext_v2i32_v2i64(ptr %A, ptr %B) nounwind {
223223
; CHECK-NEON-LABEL: smull_zext_v2i32_v2i64:
224224
; CHECK-NEON: // %bb.0:
225-
; CHECK-NEON-NEXT: ldrh w8, [x0]
226-
; CHECK-NEON-NEXT: ldrh w9, [x0, #2]
225+
; CHECK-NEON-NEXT: ldrh w8, [x0, #2]
226+
; CHECK-NEON-NEXT: ldr h0, [x0]
227227
; CHECK-NEON-NEXT: ldr d1, [x1]
228-
; CHECK-NEON-NEXT: fmov d0, x8
229-
; CHECK-NEON-NEXT: mov v0.d[1], x9
228+
; CHECK-NEON-NEXT: mov v0.d[1], x8
230229
; CHECK-NEON-NEXT: xtn v0.2s, v0.2d
231230
; CHECK-NEON-NEXT: smull v0.2d, v0.2s, v1.2s
232231
; CHECK-NEON-NEXT: ret
233232
;
234233
; CHECK-SVE-LABEL: smull_zext_v2i32_v2i64:
235234
; CHECK-SVE: // %bb.0:
236-
; CHECK-SVE-NEXT: ldrh w8, [x0]
237-
; CHECK-SVE-NEXT: ldrh w9, [x0, #2]
235+
; CHECK-SVE-NEXT: ldrh w8, [x0, #2]
236+
; CHECK-SVE-NEXT: ldr h0, [x0]
238237
; CHECK-SVE-NEXT: ldr d1, [x1]
239-
; CHECK-SVE-NEXT: fmov d0, x8
240-
; CHECK-SVE-NEXT: mov v0.d[1], x9
238+
; CHECK-SVE-NEXT: mov v0.d[1], x8
241239
; CHECK-SVE-NEXT: xtn v0.2s, v0.2d
242240
; CHECK-SVE-NEXT: smull v0.2d, v0.2s, v1.2s
243241
; CHECK-SVE-NEXT: ret

‎llvm/test/CodeGen/AArch64/dup-ext-load-combine.ll‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2+
; RUN: llc -mtriple=aarch64-none-linux-gnu < %s | FileCheck %s
3+
4+
; Test optimization of DUP with extended narrow loads
5+
; This should avoid GPR->SIMD transfers by loading directly into vector registers
6+
7+
define <4 x i32> @test_dup_zextload_i8_v4i32(ptr %p) {
8+
; CHECK-LABEL: test_dup_zextload_i8_v4i32:
9+
; CHECK: // %bb.0:
10+
; CHECK-NEXT: ldr b0, [x0]
11+
; CHECK-NEXT: dup v0.4s, v0.s[0]
12+
; CHECK-NEXT: ret
13+
%load = load i8, ptr %p, align 1
14+
%ext = zext i8 %load to i32
15+
%vec = insertelement <4 x i32> poison, i32 %ext, i32 0
16+
%dup = shufflevector <4 x i32> %vec, <4 x i32> poison, <4 x i32> zeroinitializer
17+
ret <4 x i32> %dup
18+
}
19+
20+
define <4 x i32> @test_dup_zextload_i16_v4i32(ptr %p) {
21+
; CHECK-LABEL: test_dup_zextload_i16_v4i32:
22+
; CHECK: // %bb.0:
23+
; CHECK-NEXT: ldr h0, [x0]
24+
; CHECK-NEXT: dup v0.4s, v0.s[0]
25+
; CHECK-NEXT: ret
26+
%load = load i16, ptr %p, align 2
27+
%ext = zext i16 %load to i32
28+
%vec = insertelement <4 x i32> poison, i32 %ext, i32 0
29+
%dup = shufflevector <4 x i32> %vec, <4 x i32> poison, <4 x i32> zeroinitializer
30+
ret <4 x i32> %dup
31+
}
32+
33+
define <2 x i32> @test_dup_zextload_i8_v2i32(ptr %p) {
34+
; CHECK-LABEL: test_dup_zextload_i8_v2i32:
35+
; CHECK: // %bb.0:
36+
; CHECK-NEXT: ldr b0, [x0]
37+
; CHECK-NEXT: dup v0.2s, v0.s[0]
38+
; CHECK-NEXT: ret
39+
%load = load i8, ptr %p, align 1
40+
%ext = zext i8 %load to i32
41+
%vec = insertelement <2 x i32> poison, i32 %ext, i32 0
42+
%dup = shufflevector <2 x i32> %vec, <2 x i32> poison, <2 x i32> zeroinitializer
43+
ret <2 x i32> %dup
44+
}
45+
46+
define <2 x i32> @test_dup_zextload_i16_v2i32(ptr %p) {
47+
; CHECK-LABEL: test_dup_zextload_i16_v2i32:
48+
; CHECK: // %bb.0:
49+
; CHECK-NEXT: ldr h0, [x0]
50+
; CHECK-NEXT: dup v0.2s, v0.s[0]
51+
; CHECK-NEXT: ret
52+
%load = load i16, ptr %p, align 2
53+
%ext = zext i16 %load to i32
54+
%vec = insertelement <2 x i32> poison, i32 %ext, i32 0
55+
%dup = shufflevector <2 x i32> %vec, <2 x i32> poison, <2 x i32> zeroinitializer
56+
ret <2 x i32> %dup
57+
}
58+
59+
define <8 x i16> @test_dup_zextload_i8_v8i16(ptr %p) {
60+
; CHECK-LABEL: test_dup_zextload_i8_v8i16:
61+
; CHECK: // %bb.0:
62+
; CHECK-NEXT: ldr b0, [x0]
63+
; CHECK-NEXT: dup v0.8h, v0.h[0]
64+
; CHECK-NEXT: ret
65+
%load = load i8, ptr %p, align 1
66+
%ext = zext i8 %load to i16
67+
%vec = insertelement <8 x i16> poison, i16 %ext, i32 0
68+
%dup = shufflevector <8 x i16> %vec, <8 x i16> poison, <8 x i32> zeroinitializer
69+
ret <8 x i16> %dup
70+
}
71+
72+
define <4 x i16> @test_dup_zextload_i8_v4i16(ptr %p) {
73+
; CHECK-LABEL: test_dup_zextload_i8_v4i16:
74+
; CHECK: // %bb.0:
75+
; CHECK-NEXT: ldr b0, [x0]
76+
; CHECK-NEXT: dup v0.4h, v0.h[0]
77+
; CHECK-NEXT: ret
78+
%load = load i8, ptr %p, align 1
79+
%ext = zext i8 %load to i16
80+
%vec = insertelement <4 x i16> poison, i16 %ext, i32 0
81+
%dup = shufflevector <4 x i16> %vec, <4 x i16> poison, <4 x i32> zeroinitializer
82+
ret <4 x i16> %dup
83+
}
84+
85+
define <4 x i32> @test_dup_zextload_i8_v4i32_offset(ptr %p) {
86+
; CHECK-LABEL: test_dup_zextload_i8_v4i32_offset:
87+
; CHECK: // %bb.0:
88+
; CHECK-NEXT: ldr b0, [x0, #4]
89+
; CHECK-NEXT: dup v0.4s, v0.s[0]
90+
; CHECK-NEXT: ret
91+
%addr = getelementptr inbounds i8, ptr %p, i64 4
92+
%load = load i8, ptr %addr, align 1
93+
%ext = zext i8 %load to i32
94+
%vec = insertelement <4 x i32> poison, i32 %ext, i32 0
95+
%dup = shufflevector <4 x i32> %vec, <4 x i32> poison, <4 x i32> zeroinitializer
96+
ret <4 x i32> %dup
97+
}
98+
99+
define <4 x i32> @test_dup_zextload_i16_v4i32_offset(ptr %p) {
100+
; CHECK-LABEL: test_dup_zextload_i16_v4i32_offset:
101+
; CHECK: // %bb.0:
102+
; CHECK-NEXT: ldr h0, [x0, #8]
103+
; CHECK-NEXT: dup v0.4s, v0.s[0]
104+
; CHECK-NEXT: ret
105+
%addr = getelementptr inbounds i16, ptr %p, i64 4
106+
%load = load i16, ptr %addr, align 2
107+
%ext = zext i16 %load to i32
108+
%vec = insertelement <4 x i32> poison, i32 %ext, i32 0
109+
%dup = shufflevector <4 x i32> %vec, <4 x i32> poison, <4 x i32> zeroinitializer
110+
ret <4 x i32> %dup
111+
}
112+
113+
define <4 x i32> @test_dup_zextload_i8_v4i32_reg_offset(ptr %p, i64 %offset) {
114+
; CHECK-LABEL: test_dup_zextload_i8_v4i32_reg_offset:
115+
; CHECK: // %bb.0:
116+
; CHECK-NEXT: ldr b0, [x0, x1]
117+
; CHECK-NEXT: dup v0.4s, v0.s[0]
118+
; CHECK-NEXT: ret
119+
%addr = getelementptr inbounds i8, ptr %p, i64 %offset
120+
%load = load i8, ptr %addr, align 1
121+
%ext = zext i8 %load to i32
122+
%vec = insertelement <4 x i32> poison, i32 %ext, i32 0
123+
%dup = shufflevector <4 x i32> %vec, <4 x i32> poison, <4 x i32> zeroinitializer
124+
ret <4 x i32> %dup
125+
}
126+
127+
define <4 x i32> @test_dup_zextload_i16_v4i32_reg_offset(ptr %p, i64 %offset) {
128+
; CHECK-LABEL: test_dup_zextload_i16_v4i32_reg_offset:
129+
; CHECK: // %bb.0:
130+
; CHECK-NEXT: ldr h0, [x0, x1, lsl #1]
131+
; CHECK-NEXT: dup v0.4s, v0.s[0]
132+
; CHECK-NEXT: ret
133+
%addr = getelementptr inbounds i16, ptr %p, i64 %offset
134+
%load = load i16, ptr %addr, align 2
135+
%ext = zext i16 %load to i32
136+
%vec = insertelement <4 x i32> poison, i32 %ext, i32 0
137+
%dup = shufflevector <4 x i32> %vec, <4 x i32> poison, <4 x i32> zeroinitializer
138+
ret <4 x i32> %dup
139+
}

‎llvm/test/CodeGen/AArch64/dup.ll‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@ entry:
3232
define <2 x i8> @loaddup_v2i8(ptr %p) {
3333
; CHECK-LABEL: loaddup_v2i8:
3434
; CHECK: // %bb.0: // %entry
35-
; CHECK-NEXT: ldrb w8, [x0]
36-
; CHECK-NEXT: dup v0.2s, w8
35+
; CHECK-NEXT: ldr b0, [x0]
36+
; CHECK-NEXT: dup v0.2s, v0.s[0]
3737
; CHECK-NEXT: ret
3838
entry:
3939
%a = load i8, ptr %p
@@ -189,8 +189,8 @@ entry:
189189
define <4 x i8> @loaddup_v4i8(ptr %p) {
190190
; CHECK-SD-LABEL: loaddup_v4i8:
191191
; CHECK-SD: // %bb.0: // %entry
192-
; CHECK-SD-NEXT: ldrb w8, [x0]
193-
; CHECK-SD-NEXT: dup v0.4h, w8
192+
; CHECK-SD-NEXT: ldr b0, [x0]
193+
; CHECK-SD-NEXT: dup v0.4h, v0.h[0]
194194
; CHECK-SD-NEXT: ret
195195
;
196196
; CHECK-GI-LABEL: loaddup_v4i8:
@@ -444,8 +444,8 @@ entry:
444444
define <2 x i16> @loaddup_v2i16(ptr %p) {
445445
; CHECK-SD-LABEL: loaddup_v2i16:
446446
; CHECK-SD: // %bb.0: // %entry
447-
; CHECK-SD-NEXT: ldrh w8, [x0]
448-
; CHECK-SD-NEXT: dup v0.2s, w8
447+
; CHECK-SD-NEXT: ldr h0, [x0]
448+
; CHECK-SD-NEXT: dup v0.2s, v0.s[0]
449449
; CHECK-SD-NEXT: ret
450450
;
451451
; CHECK-GI-LABEL: loaddup_v2i16:

0 commit comments

Comments
 (0)