Skip to content

Commit bcb9c9f

Browse files
momchil-velikovmomchil-velikov
committed
[AArch64] Generate zeroing forms of certain SVE2.2 instructions (1/11)
SVE2.2 introduces instructions with predicated forms with zeroing of the inactive lanes. This allows in some cases to save a `movprfx` or a `mov` instruction when emitting code for `_x` or `_z` variants of intrinsics. This patch adds support for emitting the zeroing forms of `ABS`, `NEG`, `FABS`, and `FNEG` instructions.
1 parent 4cdfa2a commit bcb9c9f

File tree

4 files changed

+720
-23
lines changed

4 files changed

+720
-23
lines changed

‎llvm/lib/Target/AArch64/AArch64InstrInfo.td‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -381,6 +381,9 @@ def NoUseScalarIncVL : Predicate<"!Subtarget->useScalarIncVL()">;
381381

382382
def UseSVEFPLD1R : Predicate<"!Subtarget->noSVEFPLD1R()">;
383383

384+
def UseUnaryUndefPseudos
385+
: Predicate<"!(Subtarget->hasSVE2() || (Subtarget->isStreaming() && Subtarget->hasSME2()))">;
386+
384387
def AArch64LocalRecover : SDNode<"ISD::LOCAL_RECOVER",
385388
SDTypeProfile<1, 1, [SDTCisSameAs<0, 1>,
386389
SDTCisInt<1>]>>;

‎llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -664,6 +664,14 @@ let Predicates = [HasSVEorSME] in {
664664
defm FABS_ZPmZ : sve_int_un_pred_arit_bitwise_fp<0b100, "fabs", AArch64fabs_mt>;
665665
defm FNEG_ZPmZ : sve_int_un_pred_arit_bitwise_fp<0b101, "fneg", AArch64fneg_mt>;
666666

667+
let Predicates = [HasSVEorSME, UseUnaryUndefPseudos] in {
668+
defm FABS_ZPmZ : sve_int_un_pred_arit_hsd<AArch64fabs_mt>;
669+
defm FNEG_ZPmZ : sve_int_un_pred_arit_hsd<AArch64fneg_mt>;
670+
671+
defm ABS_ZPmZ : sve_int_un_pred_arit_bhsd<AArch64abs_mt>;
672+
defm NEG_ZPmZ : sve_int_un_pred_arit_bhsd<AArch64neg_mt>;
673+
}
674+
667675
foreach VT = [nxv2bf16, nxv4bf16, nxv8bf16] in {
668676
// No dedicated instruction, so just clear the sign bit.
669677
def : Pat<(VT (fabs VT:$op)),
@@ -4310,16 +4318,16 @@ let Predicates = [HasSVE2p2orSME2p2] in {
43104318
defm NOT_ZPzZ : sve_int_un_pred_arit_bitwise_z<0b110, "not">;
43114319

43124320
// floating point
4313-
defm FABS_ZPzZ : sve_int_un_pred_arit_bitwise_fp_z<0b100, "fabs">;
4314-
defm FNEG_ZPzZ : sve_int_un_pred_arit_bitwise_fp_z<0b101, "fneg">;
4321+
defm FABS_ZPzZ : sve_int_un_pred_arit_bitwise_fp_z<0b100, "fabs", AArch64fabs_mt>;
4322+
defm FNEG_ZPzZ : sve_int_un_pred_arit_bitwise_fp_z<0b101, "fneg", AArch64fneg_mt>;
43154323

43164324
// SVE2p2 integer unary arithmetic, zeroing predicate
43174325
defm SXTB_ZPzZ : sve_int_un_pred_arit_h_z<0b000, "sxtb">;
43184326
defm UXTB_ZPzZ : sve_int_un_pred_arit_h_z<0b001, "uxtb">;
43194327
defm SXTH_ZPzZ : sve_int_un_pred_arit_w_z<0b010, "sxth">;
43204328
defm UXTH_ZPzZ : sve_int_un_pred_arit_w_z<0b011, "uxth">;
4321-
defm ABS_ZPzZ : sve_int_un_pred_arit_z< 0b110, "abs">;
4322-
defm NEG_ZPzZ : sve_int_un_pred_arit_z< 0b111, "neg">;
4329+
defm ABS_ZPzZ : sve_int_un_pred_arit_z< 0b110, "abs", AArch64abs_mt>;
4330+
defm NEG_ZPzZ : sve_int_un_pred_arit_z< 0b111, "neg", AArch64neg_mt>;
43234331
def SXTW_ZPzZ_D : sve_int_un_pred_arit_z<0b11, 0b1000, "sxtw", ZPR64>;
43244332
def UXTW_ZPzZ_D : sve_int_un_pred_arit_z<0b11, 0b1010, "uxtw", ZPR64>;
43254333

‎llvm/lib/Target/AArch64/SVEInstrFormats.td‎

Lines changed: 39 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -482,6 +482,8 @@ let Predicates = [HasSVEorSME] in {
482482
//===----------------------------------------------------------------------===//
483483
// SVE pattern match helpers.
484484
//===----------------------------------------------------------------------===//
485+
def SVEDup0 : ComplexPattern<vAny, 0, "SelectDupZero", []>;
486+
def SVEDup0Undef : ComplexPattern<vAny, 0, "SelectDupZeroOrUndef", []>;
485487

486488
class SVE_1_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
487489
Instruction inst>
@@ -502,6 +504,11 @@ multiclass SVE_1_Op_PassthruUndef_Pat<ValueType vtd, SDPatternOperator op, Value
502504
(inst $Op3, $Op1, $Op2)>;
503505
}
504506

507+
class SVE_1_Op_PassthruUndefZero_Pat<ValueType vtd, SDPatternOperator op, ValueType pg,
508+
ValueType vts, Instruction inst>
509+
: Pat<(vtd (op pg:$Op1, vts:$Op2, (vtd (SVEDup0Undef)))),
510+
(inst $Op1, $Op2)>;
511+
505512
// Used to match FP_ROUND_MERGE_PASSTHRU, which has an additional flag for the
506513
// type of rounding. This is matched by timm0_1 in pattern below and ignored.
507514
class SVE_1_Op_Passthru_Round_Pat<ValueType vtd, SDPatternOperator op, ValueType pg,
@@ -517,13 +524,12 @@ multiclass SVE_1_Op_PassthruUndef_Round_Pat<ValueType vtd, SDPatternOperator op,
517524
(inst $Op3, $Op1, $Op2)>;
518525
}
519526

520-
def SVEDup0 : ComplexPattern<vAny, 0, "SelectDupZero", []>;
521-
522527
class SVE_1_Op_PassthruZero_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
523528
ValueType vt2, Instruction inst>
524529
: Pat<(vtd (op (vtd (SVEDup0)), vt1:$Op1, vt2:$Op2)),
525530
(inst (IMPLICIT_DEF), $Op1, $Op2)>;
526531

532+
527533
class SVE_1_Op_Imm_OptLsl_Pat<ValueType vt, SDPatternOperator op, ZPRRegOp zprty,
528534
ValueType it, ComplexPattern cpx, Instruction inst>
529535
: Pat<(vt (op (vt zprty:$Op1), (vt (splat_vector (it (cpx i32:$imm, i32:$shift)))))),
@@ -606,8 +612,6 @@ class SVE_4_Op_Imm_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
606612
: Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3, (vt4 ImmTy:$Op4))),
607613
(inst $Op1, $Op2, $Op3, ImmTy:$Op4)>;
608614

609-
def SVEDup0Undef : ComplexPattern<vAny, 0, "SelectDupZeroOrUndef", []>;
610-
611615
let AddedComplexity = 1 in {
612616
class SVE_3_Op_Pat_SelZero<ValueType vtd, SDPatternOperator op, ValueType vt1,
613617
ValueType vt2, ValueType vt3, Instruction inst>
@@ -4820,23 +4824,18 @@ multiclass sve_int_un_pred_arit<bits<3> opc, string asm,
48204824
def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>;
48214825
def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;
48224826
def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;
4823-
4824-
def _B_UNDEF : PredOneOpPassthruPseudo<NAME # _B, ZPR8>;
4825-
def _H_UNDEF : PredOneOpPassthruPseudo<NAME # _H, ZPR16>;
4826-
def _S_UNDEF : PredOneOpPassthruPseudo<NAME # _S, ZPR32>;
4827-
def _D_UNDEF : PredOneOpPassthruPseudo<NAME # _D, ZPR64>;
4828-
4829-
defm : SVE_1_Op_PassthruUndef_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _B_UNDEF)>;
4830-
defm : SVE_1_Op_PassthruUndef_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _H_UNDEF)>;
4831-
defm : SVE_1_Op_PassthruUndef_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _S_UNDEF)>;
4832-
defm : SVE_1_Op_PassthruUndef_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _D_UNDEF)>;
48334827
}
48344828

4835-
multiclass sve_int_un_pred_arit_z<bits<3> opc, string asm> {
4829+
multiclass sve_int_un_pred_arit_z<bits<3> opc, string asm, SDPatternOperator op> {
48364830
def _B : sve_int_un_pred_arit_z<0b00, { opc, 0b0 }, asm, ZPR8>;
48374831
def _H : sve_int_un_pred_arit_z<0b01, { opc, 0b0 }, asm, ZPR16>;
48384832
def _S : sve_int_un_pred_arit_z<0b10, { opc, 0b0 }, asm, ZPR32>;
48394833
def _D : sve_int_un_pred_arit_z<0b11, { opc, 0b0 }, asm, ZPR64>;
4834+
4835+
def : SVE_1_Op_PassthruUndefZero_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;
4836+
def : SVE_1_Op_PassthruUndefZero_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>;
4837+
def : SVE_1_Op_PassthruUndefZero_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;
4838+
def : SVE_1_Op_PassthruUndefZero_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;
48404839
}
48414840

48424841
multiclass sve_int_un_pred_arit_h<bits<3> opc, string asm,
@@ -4950,7 +4949,22 @@ multiclass sve_int_un_pred_arit_bitwise_fp<bits<3> opc, string asm,
49504949
def : SVE_1_Op_Passthru_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;
49514950
def : SVE_1_Op_Passthru_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>;
49524951
def : SVE_1_Op_Passthru_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;
4952+
}
4953+
4954+
multiclass sve_int_un_pred_arit_bitwise_fp_z<bits<3> opc, string asm, SDPatternOperator op> {
4955+
def _H : sve_int_un_pred_arit_z<0b01, { opc, 0b1 }, asm, ZPR16>;
4956+
def _S : sve_int_un_pred_arit_z<0b10, { opc, 0b1 }, asm, ZPR32>;
4957+
def _D : sve_int_un_pred_arit_z<0b11, { opc, 0b1 }, asm, ZPR64>;
49534958

4959+
def : SVE_1_Op_PassthruUndefZero_Pat<nxv8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>;
4960+
def : SVE_1_Op_PassthruUndefZero_Pat<nxv4f16, op, nxv4i1, nxv4f16, !cast<Instruction>(NAME # _H)>;
4961+
def : SVE_1_Op_PassthruUndefZero_Pat<nxv2f16, op, nxv2i1, nxv2f16, !cast<Instruction>(NAME # _H)>;
4962+
def : SVE_1_Op_PassthruUndefZero_Pat<nxv4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;
4963+
def : SVE_1_Op_PassthruUndefZero_Pat<nxv2f32, op, nxv2i1, nxv2f32, !cast<Instruction>(NAME # _S)>;
4964+
def : SVE_1_Op_PassthruUndefZero_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;
4965+
}
4966+
4967+
multiclass sve_int_un_pred_arit_hsd<SDPatternOperator op> {
49544968
def _H_UNDEF : PredOneOpPassthruPseudo<NAME # _H, ZPR16>;
49554969
def _S_UNDEF : PredOneOpPassthruPseudo<NAME # _S, ZPR32>;
49564970
def _D_UNDEF : PredOneOpPassthruPseudo<NAME # _D, ZPR64>;
@@ -4963,10 +4977,16 @@ multiclass sve_int_un_pred_arit_bitwise_fp<bits<3> opc, string asm,
49634977
defm : SVE_1_Op_PassthruUndef_Pat<nxv2f64, op, nxv2i1, nxv2f64, !cast<Pseudo>(NAME # _D_UNDEF)>;
49644978
}
49654979

4966-
multiclass sve_int_un_pred_arit_bitwise_fp_z<bits<3> opc, string asm> {
4967-
def _H : sve_int_un_pred_arit_z<0b01, { opc, 0b1 }, asm, ZPR16>;
4968-
def _S : sve_int_un_pred_arit_z<0b10, { opc, 0b1 }, asm, ZPR32>;
4969-
def _D : sve_int_un_pred_arit_z<0b11, { opc, 0b1 }, asm, ZPR64>;
4980+
multiclass sve_int_un_pred_arit_bhsd<SDPatternOperator op> {
4981+
def _B_UNDEF : PredOneOpPassthruPseudo<NAME # _B, ZPR8>;
4982+
def _H_UNDEF : PredOneOpPassthruPseudo<NAME # _H, ZPR16>;
4983+
def _S_UNDEF : PredOneOpPassthruPseudo<NAME # _S, ZPR32>;
4984+
def _D_UNDEF : PredOneOpPassthruPseudo<NAME # _D, ZPR64>;
4985+
4986+
defm : SVE_1_Op_PassthruUndef_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Pseudo>(NAME # _B_UNDEF)>;
4987+
defm : SVE_1_Op_PassthruUndef_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Pseudo>(NAME # _H_UNDEF)>;
4988+
defm : SVE_1_Op_PassthruUndef_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Pseudo>(NAME # _S_UNDEF)>;
4989+
defm : SVE_1_Op_PassthruUndef_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Pseudo>(NAME # _D_UNDEF)>;
49704990
}
49714991

49724992
//===----------------------------------------------------------------------===//

0 commit comments

Comments
 (0)