Skip to content

Commit 0ecc1f0

Browse files
junparsermemfrob
junparser
authored and
memfrob
committed
[AArch64][SVEIntrinsicOpts] Convect cntb/h/w/d to vscale intrinsic or constant.
As is mentioned above Differential Revision: https://reviews.llvm.org/D104852
1 parent c7c745a commit 0ecc1f0

File tree

6 files changed

+342
-62
lines changed

6 files changed

+342
-62
lines changed

‎clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cntb.c

Lines changed: 15 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,9 @@
77
uint64_t test_svcntb()
88
{
99
// CHECK-LABEL: test_svcntb
10-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntb(i32 31)
11-
// CHECK: ret i64 %[[INTRINSIC]]
10+
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.vscale.i64()
11+
// CHECK-NEXT: %[[RET:.*]] = shl i64 %[[INTRINSIC]], 4
12+
// CHECK: ret i64 %[[RET]]
1213
return svcntb();
1314
}
1415

@@ -23,72 +24,63 @@ uint64_t test_svcntb_pat()
2324
uint64_t test_svcntb_pat_1()
2425
{
2526
// CHECK-LABEL: test_svcntb_pat_1
26-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntb(i32 1)
27-
// CHECK: ret i64 %[[INTRINSIC]]
27+
// CHECK: ret i64 1
2828
return svcntb_pat(SV_VL1);
2929
}
3030

3131
uint64_t test_svcntb_pat_2()
3232
{
3333
// CHECK-LABEL: test_svcntb_pat_2
34-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntb(i32 2)
35-
// CHECK: ret i64 %[[INTRINSIC]]
34+
// CHECK: ret i64 2
3635
return svcntb_pat(SV_VL2);
3736
}
3837

3938
uint64_t test_svcntb_pat_3()
4039
{
4140
// CHECK-LABEL: test_svcntb_pat_3
42-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntb(i32 3)
43-
// CHECK: ret i64 %[[INTRINSIC]]
41+
// CHECK: ret i64 3
4442
return svcntb_pat(SV_VL3);
4543
}
4644

4745
uint64_t test_svcntb_pat_4()
4846
{
4947
// CHECK-LABEL: test_svcntb_pat_4
50-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntb(i32 4)
51-
// CHECK: ret i64 %[[INTRINSIC]]
48+
// CHECK: ret i64 4
5249
return svcntb_pat(SV_VL4);
5350
}
5451

5552
uint64_t test_svcntb_pat_5()
5653
{
5754
// CHECK-LABEL: test_svcntb_pat_5
58-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntb(i32 5)
59-
// CHECK: ret i64 %[[INTRINSIC]]
55+
// CHECK: ret i64 5
6056
return svcntb_pat(SV_VL5);
6157
}
6258

6359
uint64_t test_svcntb_pat_6()
6460
{
6561
// CHECK-LABEL: test_svcntb_pat_6
66-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntb(i32 6)
67-
// CHECK: ret i64 %[[INTRINSIC]]
62+
// CHECK: ret i64 6
6863
return svcntb_pat(SV_VL6);
6964
}
7065

7166
uint64_t test_svcntb_pat_7()
7267
{
7368
// CHECK-LABEL: test_svcntb_pat_7
74-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntb(i32 7)
75-
// CHECK: ret i64 %[[INTRINSIC]]
69+
// CHECK: ret i64 7
7670
return svcntb_pat(SV_VL7);
7771
}
7872

7973
uint64_t test_svcntb_pat_8()
8074
{
8175
// CHECK-LABEL: test_svcntb_pat_8
82-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntb(i32 8)
83-
// CHECK: ret i64 %[[INTRINSIC]]
76+
// CHECK: ret i64 8
8477
return svcntb_pat(SV_VL8);
8578
}
8679

8780
uint64_t test_svcntb_pat_9()
8881
{
8982
// CHECK-LABEL: test_svcntb_pat_9
90-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntb(i32 9)
91-
// CHECK: ret i64 %[[INTRINSIC]]
83+
// CHECK: ret i64 16
9284
return svcntb_pat(SV_VL16);
9385
}
9486

@@ -143,7 +135,8 @@ uint64_t test_svcntb_pat_15()
143135
uint64_t test_svcntb_pat_16()
144136
{
145137
// CHECK-LABEL: test_svcntb_pat_16
146-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntb(i32 31)
147-
// CHECK: ret i64 %[[INTRINSIC]]
138+
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.vscale.i64()
139+
// CHECK-NEXT: %[[RET:.*]] = shl i64 %[[INTRINSIC]], 4
140+
// CHECK: ret i64 %[[RET]]
148141
return svcntb_pat(SV_ALL);
149142
}

‎clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cntd.c

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,9 @@
77
uint64_t test_svcntd()
88
{
99
// CHECK-LABEL: test_svcntd
10-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntd(i32 31)
11-
// CHECK: ret i64 %[[INTRINSIC]]
10+
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.vscale.i64()
11+
// CHECK-NEXT: %[[RET:.*]] = shl i64 %[[INTRINSIC]], 1
12+
// CHECK: ret i64 %[[RET]]
1213
return svcntd();
1314
}
1415

@@ -23,16 +24,14 @@ uint64_t test_svcntd_pat()
2324
uint64_t test_svcntd_pat_1()
2425
{
2526
// CHECK-LABEL: test_svcntd_pat_1
26-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntd(i32 1)
27-
// CHECK: ret i64 %[[INTRINSIC]]
27+
// CHECK: ret i64 1
2828
return svcntd_pat(SV_VL1);
2929
}
3030

3131
uint64_t test_svcntd_pat_2()
3232
{
3333
// CHECK-LABEL: test_svcntd_pat_2
34-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntd(i32 2)
35-
// CHECK: ret i64 %[[INTRINSIC]]
34+
// CHECK: ret i64 2
3635
return svcntd_pat(SV_VL2);
3736
}
3837

@@ -143,7 +142,8 @@ uint64_t test_svcntd_pat_15()
143142
uint64_t test_svcntd_pat_16()
144143
{
145144
// CHECK-LABEL: test_svcntd_pat_16
146-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntd(i32 31)
147-
// CHECK: ret i64 %[[INTRINSIC]]
145+
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.vscale.i64()
146+
// CHECK-NEXT: %[[RET:.*]] = shl i64 %[[INTRINSIC]], 1
147+
// CHECK: ret i64 %[[RET]]
148148
return svcntd_pat(SV_ALL);
149149
}

‎clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cnth.c

Lines changed: 14 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,9 @@
77
uint64_t test_svcnth()
88
{
99
// CHECK-LABEL: test_svcnth
10-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cnth(i32 31)
11-
// CHECK: ret i64 %[[INTRINSIC]]
10+
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.vscale.i64()
11+
// CHECK-NEXT: %[[RET:.*]] = shl i64 %[[INTRINSIC]], 3
12+
// CHECK: ret i64 %[[RET]]
1213
return svcnth();
1314
}
1415

@@ -23,64 +24,56 @@ uint64_t test_svcnth_pat()
2324
uint64_t test_svcnth_pat_1()
2425
{
2526
// CHECK-LABEL: test_svcnth_pat_1
26-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cnth(i32 1)
27-
// CHECK: ret i64 %[[INTRINSIC]]
27+
// CHECK: ret i64 1
2828
return svcnth_pat(SV_VL1);
2929
}
3030

3131
uint64_t test_svcnth_pat_2()
3232
{
3333
// CHECK-LABEL: test_svcnth_pat_2
34-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cnth(i32 2)
35-
// CHECK: ret i64 %[[INTRINSIC]]
34+
// CHECK: ret i64 2
3635
return svcnth_pat(SV_VL2);
3736
}
3837

3938
uint64_t test_svcnth_pat_3()
4039
{
4140
// CHECK-LABEL: test_svcnth_pat_3
42-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cnth(i32 3)
43-
// CHECK: ret i64 %[[INTRINSIC]]
41+
// CHECK: ret i64 3
4442
return svcnth_pat(SV_VL3);
4543
}
4644

4745
uint64_t test_svcnth_pat_4()
4846
{
4947
// CHECK-LABEL: test_svcnth_pat_4
50-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cnth(i32 4)
51-
// CHECK: ret i64 %[[INTRINSIC]]
48+
// CHECK: ret i64 4
5249
return svcnth_pat(SV_VL4);
5350
}
5451

5552
uint64_t test_svcnth_pat_5()
5653
{
5754
// CHECK-LABEL: test_svcnth_pat_5
58-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cnth(i32 5)
59-
// CHECK: ret i64 %[[INTRINSIC]]
55+
// CHECK: ret i64 5
6056
return svcnth_pat(SV_VL5);
6157
}
6258

6359
uint64_t test_svcnth_pat_6()
6460
{
6561
// CHECK-LABEL: test_svcnth_pat_6
66-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cnth(i32 6)
67-
// CHECK: ret i64 %[[INTRINSIC]]
62+
// CHECK: ret i64 6
6863
return svcnth_pat(SV_VL6);
6964
}
7065

7166
uint64_t test_svcnth_pat_7()
7267
{
7368
// CHECK-LABEL: test_svcnth_pat_7
74-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cnth(i32 7)
75-
// CHECK: ret i64 %[[INTRINSIC]]
69+
// CHECK: ret i64 7
7670
return svcnth_pat(SV_VL7);
7771
}
7872

7973
uint64_t test_svcnth_pat_8()
8074
{
8175
// CHECK-LABEL: test_svcnth_pat_8
82-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cnth(i32 8)
83-
// CHECK: ret i64 %[[INTRINSIC]]
76+
// CHECK: ret i64 8
8477
return svcnth_pat(SV_VL8);
8578
}
8679

@@ -143,7 +136,8 @@ uint64_t test_svcnth_pat_15()
143136
uint64_t test_svcnth_pat_16()
144137
{
145138
// CHECK-LABEL: test_svcnth_pat_16
146-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cnth(i32 31)
147-
// CHECK: ret i64 %[[INTRINSIC]]
139+
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.vscale.i64()
140+
// CHECK-NEXT: %[[RET:.*]] = shl i64 %[[INTRINSIC]], 3
141+
// CHECK: ret i64 %[[RET]]
148142
return svcnth_pat(SV_ALL);
149143
}

‎clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_cntw.c

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,9 @@
77
uint64_t test_svcntw()
88
{
99
// CHECK-LABEL: test_svcntw
10-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntw(i32 31)
11-
// CHECK: ret i64 %[[INTRINSIC]]
10+
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.vscale.i64()
11+
// CHECK-NEXT: %[[RET:.*]] = shl i64 %[[INTRINSIC]], 2
12+
// CHECK: ret i64 %[[RET]]
1213
return svcntw();
1314
}
1415

@@ -23,32 +24,28 @@ uint64_t test_svcntw_pat()
2324
uint64_t test_svcntw_pat_1()
2425
{
2526
// CHECK-LABEL: test_svcntw_pat_1
26-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntw(i32 1)
27-
// CHECK: ret i64 %[[INTRINSIC]]
27+
// CHECK: ret i64 1
2828
return svcntw_pat(SV_VL1);
2929
}
3030

3131
uint64_t test_svcntw_pat_2()
3232
{
3333
// CHECK-LABEL: test_svcntw_pat_2
34-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntw(i32 2)
35-
// CHECK: ret i64 %[[INTRINSIC]]
34+
// CHECK: ret i64 2
3635
return svcntw_pat(SV_VL2);
3736
}
3837

3938
uint64_t test_svcntw_pat_3()
4039
{
4140
// CHECK-LABEL: test_svcntw_pat_3
42-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntw(i32 3)
43-
// CHECK: ret i64 %[[INTRINSIC]]
41+
// CHECK: ret i64 3
4442
return svcntw_pat(SV_VL3);
4543
}
4644

4745
uint64_t test_svcntw_pat_4()
4846
{
4947
// CHECK-LABEL: test_svcntw_pat_4
50-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntw(i32 4)
51-
// CHECK: ret i64 %[[INTRINSIC]]
48+
// CHECK: ret i64 4
5249
return svcntw_pat(SV_VL4);
5350
}
5451

@@ -143,7 +140,8 @@ uint64_t test_svcntw_pat_15()
143140
uint64_t test_svcntw_pat_16()
144141
{
145142
// CHECK-LABEL: test_svcntw_pat_16
146-
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.aarch64.sve.cntw(i32 31)
147-
// CHECK: ret i64 %[[INTRINSIC]]
143+
// CHECK: %[[INTRINSIC:.*]] = call i64 @llvm.vscale.i64()
144+
// CHECK-NEXT: %[[RET:.*]] = shl i64 %[[INTRINSIC]], 2
145+
// CHECK: ret i64 %[[RET]]
148146
return svcntw_pat(SV_ALL);
149147
}

‎llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -642,6 +642,46 @@ static Optional<Instruction *> instCombineRDFFR(InstCombiner &IC,
642642
return IC.replaceInstUsesWith(II, RDFFR);
643643
}
644644

645+
static Optional<Instruction *>
646+
instCombineSVECntElts(InstCombiner &IC, IntrinsicInst &II, unsigned NumElts) {
647+
const auto Pattern = cast<ConstantInt>(II.getArgOperand(0))->getZExtValue();
648+
649+
if (Pattern == AArch64SVEPredPattern::all) {
650+
LLVMContext &Ctx = II.getContext();
651+
IRBuilder<> Builder(Ctx);
652+
Builder.SetInsertPoint(&II);
653+
654+
Constant *StepVal = ConstantInt::get(II.getType(), NumElts);
655+
auto *VScale = Builder.CreateVScale(StepVal);
656+
VScale->takeName(&II);
657+
return IC.replaceInstUsesWith(II, VScale);
658+
}
659+
660+
unsigned MinNumElts = 0;
661+
switch (Pattern) {
662+
default:
663+
return None;
664+
case AArch64SVEPredPattern::vl1:
665+
case AArch64SVEPredPattern::vl2:
666+
case AArch64SVEPredPattern::vl3:
667+
case AArch64SVEPredPattern::vl4:
668+
case AArch64SVEPredPattern::vl5:
669+
case AArch64SVEPredPattern::vl6:
670+
case AArch64SVEPredPattern::vl7:
671+
case AArch64SVEPredPattern::vl8:
672+
MinNumElts = Pattern;
673+
break;
674+
case AArch64SVEPredPattern::vl16:
675+
MinNumElts = 16;
676+
break;
677+
}
678+
679+
return NumElts >= MinNumElts
680+
? Optional<Instruction *>(IC.replaceInstUsesWith(
681+
II, ConstantInt::get(II.getType(), MinNumElts)))
682+
: None;
683+
}
684+
645685
Optional<Instruction *>
646686
AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC,
647687
IntrinsicInst &II) const {
@@ -661,6 +701,14 @@ AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC,
661701
case Intrinsic::aarch64_sve_lasta:
662702
case Intrinsic::aarch64_sve_lastb:
663703
return instCombineSVELast(IC, II);
704+
case Intrinsic::aarch64_sve_cntd:
705+
return instCombineSVECntElts(IC, II, 2);
706+
case Intrinsic::aarch64_sve_cntw:
707+
return instCombineSVECntElts(IC, II, 4);
708+
case Intrinsic::aarch64_sve_cnth:
709+
return instCombineSVECntElts(IC, II, 8);
710+
case Intrinsic::aarch64_sve_cntb:
711+
return instCombineSVECntElts(IC, II, 16);
664712
}
665713

666714
return None;

0 commit comments

Comments
 (0)