Skip to content

Commit 61cdba6

Browse files
davemgreendavemgreen
authored
[AArch64][GlobalISel] Expand 64bit extracts to 128bit to allow more patterns (llvm#142904)
SDAG will promote a 64bit G_EXTRACT_VECTOR_ELT to 128 to reduce the number of duplicate lane-extract patterns needed. This patch does the same for gisel inside regbankselect, so that selection will operate on the larger vectors. Most of the tests just add kill markers, but arm64-neon-v8.1a.ll shows the lanewise patterns now being selected (which, as of this patch is now fully producing the same code as SDAG).
1 parent 6559831 commit 61cdba6

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+334
-592
lines changed

‎llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -399,6 +399,26 @@ void AArch64RegisterBankInfo::applyMappingImpl(
399399
MI.getOperand(1).setReg(ConstReg);
400400
return applyDefaultMapping(OpdMapper);
401401
}
402+
case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
403+
// SDAG will promote a 64bit G_EXTRACT_VECTOR_ELT to 128 to reduce the
404+
// number of duplicate lane-extract patterns needed. Do the same here so
405+
// that selection will operate on the larger vectors.
406+
Register Src = MI.getOperand(1).getReg();
407+
LLT SrcTy = MRI.getType(Src);
408+
assert(SrcTy.getSizeInBits() == 64 && "Expected 64-bit source vector");
409+
LLT DstTy = SrcTy.multiplyElements(2);
410+
Builder.setInsertPt(*MI.getParent(), MI.getIterator());
411+
auto Undef = Builder.buildUndef(SrcTy);
412+
auto Concat = Builder.buildConcatVectors(DstTy, {Src, Undef.getReg(0)});
413+
MRI.setRegBank(Undef.getReg(0), getRegBank(AArch64::FPRRegBankID));
414+
MRI.setRegBank(Concat.getReg(0), getRegBank(AArch64::FPRRegBankID));
415+
for (MachineInstr &Ext :
416+
make_early_inc_range(MRI.use_nodbg_instructions(Src))) {
417+
if (Ext.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT)
418+
Ext.getOperand(1).setReg(Concat.getReg(0));
419+
}
420+
return applyDefaultMapping(OpdMapper);
421+
}
402422
default:
403423
llvm_unreachable("Don't know how to handle that operation");
404424
}
@@ -1014,14 +1034,20 @@ AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
10141034
}
10151035
break;
10161036
}
1017-
case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1037+
case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
10181038
// Destination and source need to be FPRs.
10191039
OpRegBankIdx[0] = PMI_FirstFPR;
10201040
OpRegBankIdx[1] = PMI_FirstFPR;
1021-
1022-
// Index needs to be a GPR.
1041+
// Index needs to be a GPR constant.
10231042
OpRegBankIdx[2] = PMI_FirstGPR;
1043+
// SDAG will promote a 64bit G_EXTRACT_VECTOR_ELT to 128 to reduce the
1044+
// number of duplicate lane-extract patterns needed. Do the same here so
1045+
// that selection will operate on the larger vectors.
1046+
LLT Ty = MRI.getType(MI.getOperand(1).getReg());
1047+
if (!Ty.isScalable() && Ty.getSizeInBits() == 64)
1048+
MappingID = CustomMappingID;
10241049
break;
1050+
}
10251051
case TargetOpcode::G_INSERT_VECTOR_ELT:
10261052
OpRegBankIdx[0] = PMI_FirstFPR;
10271053
OpRegBankIdx[1] = PMI_FirstFPR;

‎llvm/test/CodeGen/AArch64/GlobalISel/regbank-extract-vector-elt.mir‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,9 @@ body: |
9494
; CHECK-NEXT: {{ $}}
9595
; CHECK-NEXT: [[COPY:%[0-9]+]]:fpr(<4 x s16>) = COPY $d0
9696
; CHECK-NEXT: [[C:%[0-9]+]]:gpr(s64) = G_CONSTANT i64 1
97-
; CHECK-NEXT: [[EVEC:%[0-9]+]]:fpr(s16) = G_EXTRACT_VECTOR_ELT [[COPY]](<4 x s16>), [[C]](s64)
97+
; CHECK-NEXT: [[DEF:%[0-9]+]]:fpr(<4 x s16>) = G_IMPLICIT_DEF
98+
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:fpr(<8 x s16>) = G_CONCAT_VECTORS [[COPY]](<4 x s16>), [[DEF]](<4 x s16>)
99+
; CHECK-NEXT: [[EVEC:%[0-9]+]]:fpr(s16) = G_EXTRACT_VECTOR_ELT [[CONCAT_VECTORS]](<8 x s16>), [[C]](s64)
98100
; CHECK-NEXT: $h0 = COPY [[EVEC]](s16)
99101
; CHECK-NEXT: RET_ReallyLR implicit $h0
100102
%0:_(<4 x s16>) = COPY $d0

‎llvm/test/CodeGen/AArch64/aarch64-bif-gen.ll‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,9 @@ define <1 x i32> @test_bitf_v1i32(<1 x i32> %A, <1 x i32> %B, <1 x i32> %C) {
7070
;
7171
; CHECK-GI-LABEL: test_bitf_v1i32:
7272
; CHECK-GI: // %bb.0:
73+
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
74+
; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q1
75+
; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q2
7376
; CHECK-GI-NEXT: fmov w8, s2
7477
; CHECK-GI-NEXT: fmov w9, s1
7578
; CHECK-GI-NEXT: fmov w10, s0

‎llvm/test/CodeGen/AArch64/aarch64-bit-gen.ll‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,9 @@ define <1 x i32> @test_bit_v1i32(<1 x i32> %A, <1 x i32> %B, <1 x i32> %C) {
7070
;
7171
; CHECK-GI-LABEL: test_bit_v1i32:
7272
; CHECK-GI: // %bb.0:
73+
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
74+
; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q1
75+
; CHECK-GI-NEXT: // kill: def $d2 killed $d2 def $q2
7376
; CHECK-GI-NEXT: fmov w8, s2
7477
; CHECK-GI-NEXT: fmov w9, s1
7578
; CHECK-GI-NEXT: fmov w10, s0

‎llvm/test/CodeGen/AArch64/abs.ll‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,7 @@ define <1 x i32> @abs_v1i32(<1 x i32> %a){
243243
;
244244
; CHECK-GI-LABEL: abs_v1i32:
245245
; CHECK-GI: // %bb.0: // %entry
246+
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
246247
; CHECK-GI-NEXT: fmov w8, s0
247248
; CHECK-GI-NEXT: fmov w9, s0
248249
; CHECK-GI-NEXT: cmp w8, #0

‎llvm/test/CodeGen/AArch64/arm64-neon-copy.ll‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1215,6 +1215,7 @@ define <8 x i8> @testDUP.v1i8(<1 x i8> %a) {
12151215
;
12161216
; CHECK-GI-LABEL: testDUP.v1i8:
12171217
; CHECK-GI: // %bb.0:
1218+
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
12181219
; CHECK-GI-NEXT: fmov w8, s0
12191220
; CHECK-GI-NEXT: dup v0.8b, w8
12201221
; CHECK-GI-NEXT: ret
@@ -1710,7 +1711,7 @@ define <16 x i8> @test_concat_v16i8_v8i8_v16i8(<8 x i8> %x, <16 x i8> %y) #0 {
17101711
; CHECK-GI-NEXT: mov v2.16b, v1.16b
17111712
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
17121713
; CHECK-GI-NEXT: adrp x8, .LCPI127_0
1713-
; CHECK-GI-NEXT: mov v1.b[0], v0.b[0]
1714+
; CHECK-GI-NEXT: mov b1, v0.b[0]
17141715
; CHECK-GI-NEXT: mov v1.b[1], v0.b[1]
17151716
; CHECK-GI-NEXT: mov v1.b[2], v0.b[2]
17161717
; CHECK-GI-NEXT: mov v1.b[3], v0.b[3]
@@ -1817,7 +1818,7 @@ define <16 x i8> @test_concat_v16i8_v8i8_v8i8(<8 x i8> %x, <8 x i8> %y) #0 {
18171818
; CHECK-GI-LABEL: test_concat_v16i8_v8i8_v8i8:
18181819
; CHECK-GI: // %bb.0: // %entry
18191820
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
1820-
; CHECK-GI-NEXT: mov v2.b[0], v0.b[0]
1821+
; CHECK-GI-NEXT: mov b2, v0.b[0]
18211822
; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q1
18221823
; CHECK-GI-NEXT: mov v2.b[1], v0.b[1]
18231824
; CHECK-GI-NEXT: mov v2.b[2], v0.b[2]
@@ -1903,7 +1904,7 @@ define <8 x i16> @test_concat_v8i16_v4i16_v8i16(<4 x i16> %x, <8 x i16> %y) #0 {
19031904
; CHECK-GI-NEXT: mov v2.16b, v1.16b
19041905
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
19051906
; CHECK-GI-NEXT: adrp x8, .LCPI131_0
1906-
; CHECK-GI-NEXT: mov v1.h[0], v0.h[0]
1907+
; CHECK-GI-NEXT: mov h1, v0.h[0]
19071908
; CHECK-GI-NEXT: mov v1.h[1], v0.h[1]
19081909
; CHECK-GI-NEXT: mov v1.h[2], v0.h[2]
19091910
; CHECK-GI-NEXT: mov v1.h[3], v0.h[3]
@@ -1974,7 +1975,7 @@ define <8 x i16> @test_concat_v8i16_v4i16_v4i16(<4 x i16> %x, <4 x i16> %y) #0 {
19741975
; CHECK-GI-LABEL: test_concat_v8i16_v4i16_v4i16:
19751976
; CHECK-GI: // %bb.0: // %entry
19761977
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
1977-
; CHECK-GI-NEXT: mov v2.h[0], v0.h[0]
1978+
; CHECK-GI-NEXT: mov h2, v0.h[0]
19781979
; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q1
19791980
; CHECK-GI-NEXT: mov v2.h[1], v0.h[1]
19801981
; CHECK-GI-NEXT: mov v2.h[2], v0.h[2]
@@ -2036,7 +2037,7 @@ define <4 x i32> @test_concat_v4i32_v2i32_v4i32(<2 x i32> %x, <4 x i32> %y) #0 {
20362037
; CHECK-GI-NEXT: mov v2.16b, v1.16b
20372038
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
20382039
; CHECK-GI-NEXT: adrp x8, .LCPI135_0
2039-
; CHECK-GI-NEXT: mov v1.s[0], v0.s[0]
2040+
; CHECK-GI-NEXT: mov s1, v0.s[0]
20402041
; CHECK-GI-NEXT: mov v1.s[1], v0.s[1]
20412042
; CHECK-GI-NEXT: ldr q0, [x8, :lo12:.LCPI135_0]
20422043
; CHECK-GI-NEXT: tbl v0.16b, { v1.16b, v2.16b }, v0.16b
@@ -2242,6 +2243,7 @@ define <8 x i8> @concat_vector_v8i8(<1 x i8> %a) {
22422243
;
22432244
; CHECK-GI-LABEL: concat_vector_v8i8:
22442245
; CHECK-GI: // %bb.0:
2246+
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
22452247
; CHECK-GI-NEXT: fmov w8, s0
22462248
; CHECK-GI-NEXT: dup v0.8b, w8
22472249
; CHECK-GI-NEXT: ret
@@ -2268,6 +2270,7 @@ define <16 x i8> @concat_vector_v16i8(<1 x i8> %a) {
22682270
;
22692271
; CHECK-GI-LABEL: concat_vector_v16i8:
22702272
; CHECK-GI: // %bb.0:
2273+
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
22712274
; CHECK-GI-NEXT: fmov w8, s0
22722275
; CHECK-GI-NEXT: dup v0.16b, w8
22732276
; CHECK-GI-NEXT: ret

‎llvm/test/CodeGen/AArch64/arm64-neon-simd-ldst-one.ll‎

Lines changed: 15 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -614,16 +614,11 @@ entry:
614614
}
615615

616616
define void @test_vst1_lane0_s16(ptr %a, <4 x i16> %b) {
617-
; CHECK-GI-LABEL: test_vst1_lane0_s16:
618-
; CHECK-GI: // %bb.0: // %entry
619-
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
620-
; CHECK-GI-NEXT: str h0, [x0]
621-
; CHECK-GI-NEXT: ret
622-
;
623-
; CHECK-SD-LABEL: test_vst1_lane0_s16:
624-
; CHECK-SD: // %bb.0: // %entry
625-
; CHECK-SD-NEXT: str h0, [x0]
626-
; CHECK-SD-NEXT: ret
617+
; CHECK-LABEL: test_vst1_lane0_s16:
618+
; CHECK: // %bb.0: // %entry
619+
; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
620+
; CHECK-NEXT: str h0, [x0]
621+
; CHECK-NEXT: ret
627622
entry:
628623
%0 = extractelement <4 x i16> %b, i32 0
629624
store i16 %0, ptr %a, align 2
@@ -643,16 +638,11 @@ entry:
643638
}
644639

645640
define void @test_vst1_lane0_s32(ptr %a, <2 x i32> %b) {
646-
; CHECK-GI-LABEL: test_vst1_lane0_s32:
647-
; CHECK-GI: // %bb.0: // %entry
648-
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
649-
; CHECK-GI-NEXT: str s0, [x0]
650-
; CHECK-GI-NEXT: ret
651-
;
652-
; CHECK-SD-LABEL: test_vst1_lane0_s32:
653-
; CHECK-SD: // %bb.0: // %entry
654-
; CHECK-SD-NEXT: str s0, [x0]
655-
; CHECK-SD-NEXT: ret
641+
; CHECK-LABEL: test_vst1_lane0_s32:
642+
; CHECK: // %bb.0: // %entry
643+
; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
644+
; CHECK-NEXT: str s0, [x0]
645+
; CHECK-NEXT: ret
656646
entry:
657647
%0 = extractelement <2 x i32> %b, i32 0
658648
store i32 %0, ptr %a, align 4
@@ -683,16 +673,11 @@ entry:
683673
}
684674

685675
define void @test_vst1_lane0_f32(ptr %a, <2 x float> %b) {
686-
; CHECK-GI-LABEL: test_vst1_lane0_f32:
687-
; CHECK-GI: // %bb.0: // %entry
688-
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
689-
; CHECK-GI-NEXT: str s0, [x0]
690-
; CHECK-GI-NEXT: ret
691-
;
692-
; CHECK-SD-LABEL: test_vst1_lane0_f32:
693-
; CHECK-SD: // %bb.0: // %entry
694-
; CHECK-SD-NEXT: str s0, [x0]
695-
; CHECK-SD-NEXT: ret
676+
; CHECK-LABEL: test_vst1_lane0_f32:
677+
; CHECK: // %bb.0: // %entry
678+
; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
679+
; CHECK-NEXT: str s0, [x0]
680+
; CHECK-NEXT: ret
696681
entry:
697682
%0 = extractelement <2 x float> %b, i32 0
698683
store float %0, ptr %a, align 4

‎llvm/test/CodeGen/AArch64/arm64-neon-v8.1a.ll‎

Lines changed: 19 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -663,24 +663,14 @@ entry:
663663
}
664664

665665
define i32 @test_vqrdmlahs_lane_s32(i32 %a, i32 %b, <2 x i32> %c) {
666-
; CHECK-SD-LABEL: test_vqrdmlahs_lane_s32:
667-
; CHECK-SD: // %bb.0: // %entry
668-
; CHECK-SD-NEXT: fmov s1, w0
669-
; CHECK-SD-NEXT: fmov s2, w1
670-
; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
671-
; CHECK-SD-NEXT: sqrdmlah s1, s2, v0.s[1]
672-
; CHECK-SD-NEXT: fmov w0, s1
673-
; CHECK-SD-NEXT: ret
674-
;
675-
; CHECK-GI-LABEL: test_vqrdmlahs_lane_s32:
676-
; CHECK-GI: // %bb.0: // %entry
677-
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
678-
; CHECK-GI-NEXT: fmov s1, w0
679-
; CHECK-GI-NEXT: fmov s2, w1
680-
; CHECK-GI-NEXT: mov s0, v0.s[1]
681-
; CHECK-GI-NEXT: sqrdmlah s1, s2, s0
682-
; CHECK-GI-NEXT: fmov w0, s1
683-
; CHECK-GI-NEXT: ret
666+
; CHECK-LABEL: test_vqrdmlahs_lane_s32:
667+
; CHECK: // %bb.0: // %entry
668+
; CHECK-NEXT: fmov s1, w0
669+
; CHECK-NEXT: fmov s2, w1
670+
; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
671+
; CHECK-NEXT: sqrdmlah s1, s2, v0.s[1]
672+
; CHECK-NEXT: fmov w0, s1
673+
; CHECK-NEXT: ret
684674
entry:
685675
%vget_lane = extractelement <2 x i32> %c, i64 1
686676
%vqrdmlahs_s32.i = tail call i32 @llvm.aarch64.neon.sqrdmlah.i32(i32 %a, i32 %b, i32 %vget_lane) #4
@@ -813,24 +803,14 @@ entry:
813803
}
814804

815805
define i32 @test_vqrdmlshs_lane_s32(i32 %a, i32 %b, <2 x i32> %c) {
816-
; CHECK-SD-LABEL: test_vqrdmlshs_lane_s32:
817-
; CHECK-SD: // %bb.0: // %entry
818-
; CHECK-SD-NEXT: fmov s1, w0
819-
; CHECK-SD-NEXT: fmov s2, w1
820-
; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
821-
; CHECK-SD-NEXT: sqrdmlsh s1, s2, v0.s[1]
822-
; CHECK-SD-NEXT: fmov w0, s1
823-
; CHECK-SD-NEXT: ret
824-
;
825-
; CHECK-GI-LABEL: test_vqrdmlshs_lane_s32:
826-
; CHECK-GI: // %bb.0: // %entry
827-
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
828-
; CHECK-GI-NEXT: fmov s1, w0
829-
; CHECK-GI-NEXT: fmov s2, w1
830-
; CHECK-GI-NEXT: mov s0, v0.s[1]
831-
; CHECK-GI-NEXT: sqrdmlsh s1, s2, s0
832-
; CHECK-GI-NEXT: fmov w0, s1
833-
; CHECK-GI-NEXT: ret
806+
; CHECK-LABEL: test_vqrdmlshs_lane_s32:
807+
; CHECK: // %bb.0: // %entry
808+
; CHECK-NEXT: fmov s1, w0
809+
; CHECK-NEXT: fmov s2, w1
810+
; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
811+
; CHECK-NEXT: sqrdmlsh s1, s2, v0.s[1]
812+
; CHECK-NEXT: fmov w0, s1
813+
; CHECK-NEXT: ret
834814
entry:
835815
%vget_lane = extractelement <2 x i32> %c, i64 1
836816
%vqrdmlshs_s32.i = tail call i32 @llvm.aarch64.neon.sqrdmlsh.i32(i32 %a, i32 %b, i32 %vget_lane) #4
@@ -867,3 +847,6 @@ entry:
867847
%vqrdmlshs_s32.i = tail call i32 @llvm.aarch64.neon.sqrdmlsh.i32(i32 %a, i32 %b, i32 %vgetq_lane) #4
868848
ret i32 %vqrdmlshs_s32.i
869849
}
850+
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
851+
; CHECK-GI: {{.*}}
852+
; CHECK-SD: {{.*}}

‎llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -271,6 +271,7 @@ define half @test_vcvt_f16_f32(<1 x float> %x) {
271271
;
272272
; GISEL-LABEL: test_vcvt_f16_f32:
273273
; GISEL: // %bb.0:
274+
; GISEL-NEXT: // kill: def $d0 killed $d0 def $q0
274275
; GISEL-NEXT: fcvt h0, s0
275276
; GISEL-NEXT: ret
276277
%tmp = fptrunc <1 x float> %x to <1 x half>

‎llvm/test/CodeGen/AArch64/bswap.ll‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,7 @@ define <1 x i32> @bswap_v1i32(<1 x i32> %a){
207207
;
208208
; CHECK-GI-LABEL: bswap_v1i32:
209209
; CHECK-GI: // %bb.0: // %entry
210+
; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0
210211
; CHECK-GI-NEXT: fmov w8, s0
211212
; CHECK-GI-NEXT: rev w8, w8
212213
; CHECK-GI-NEXT: fmov s0, w8

0 commit comments

Comments
 (0)