@@ -704,10 +704,15 @@ def topbitsallzero64: PatLeaf<(i64 GPR64:$src), [{
704704 }]>;
705705
706706// Node definitions.
707+ // Compare-and-branch
707708def AArch64CB : SDNode<"AArch64ISD::CB", SDT_AArch64cb, [SDNPHasChain]>;
709+ // Page address of a TargetGlobalAddress operand.
708710def AArch64adrp : SDNode<"AArch64ISD::ADRP", SDTIntUnaryOp, []>;
709711def AArch64adr : SDNode<"AArch64ISD::ADR", SDTIntUnaryOp, []>;
712+ // Add the low 12 bits of a TargetGlobalAddress operand.
710713def AArch64addlow : SDNode<"AArch64ISD::ADDlow", SDTIntBinOp, []>;
714+ // Load from automatically generated descriptor (e.g. Global Offset Table, TLS
715+ // record).
711716def AArch64LOADgot : SDNode<"AArch64ISD::LOADgot", SDTIntUnaryOp>;
712717def AArch64callseq_start : SDNode<"ISD::CALLSEQ_START",
713718 SDCallSeqStart<[ SDTCisVT<0, i32>,
@@ -722,21 +727,27 @@ def AArch64call : SDNode<"AArch64ISD::CALL",
722727 [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
723728 SDNPVariadic]>;
724729
730+ // Function call followed by a BTI instruction.
725731def AArch64call_bti : SDNode<"AArch64ISD::CALL_BTI",
726732 SDTypeProfile<0, -1, [SDTCisPtrTy<0>]>,
727733 [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
728734 SDNPVariadic]>;
729735
736+ // Pseudo for a OBJC call that gets emitted together with a special `mov
737+ // x29, x29` marker instruction.
730738def AArch64call_rvmarker: SDNode<"AArch64ISD::CALL_RVMARKER",
731739 SDTypeProfile<0, -1, [SDTCisPtrTy<0>]>,
732740 [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
733741 SDNPVariadic]>;
734742
743+ // A call with the callee in x16, i.e. "blr x16".
735744def AArch64call_arm64ec_to_x64 : SDNode<"AArch64ISD::CALL_ARM64EC_TO_X64",
736745 SDTypeProfile<0, -1, [SDTCisPtrTy<0>]>,
737746 [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
738747 SDNPVariadic]>;
739748
749+ // Function call, authenticating the callee value first:
750+ // AUTH_CALL chain, callee, auth key #, int disc, addr disc, operands.
740751def AArch64authcall : SDNode<"AArch64ISD::AUTH_CALL",
741752 SDTypeProfile<0, -1, [SDTCisPtrTy<0>,
742753 SDTCisVT<1, i32>,
@@ -745,13 +756,16 @@ def AArch64authcall : SDNode<"AArch64ISD::AUTH_CALL",
745756 [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
746757 SDNPVariadic]>;
747758
759+ // AUTH_TC_RETURN chain, callee, fpdiff, auth key #, int disc, addr disc,
760+ // operands.
748761def AArch64authtcret: SDNode<"AArch64ISD::AUTH_TC_RETURN",
749762 SDTypeProfile<0, 5, [SDTCisPtrTy<0>,
750763 SDTCisVT<2, i32>,
751764 SDTCisVT<3, i64>,
752765 SDTCisVT<4, i64>]>,
753766 [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>;
754767
768+ // Authenticated variant of CALL_RVMARKER.
755769def AArch64authcall_rvmarker : SDNode<"AArch64ISD::AUTH_CALL_RVMARKER",
756770 SDTypeProfile<0, -1, [SDTCisPtrTy<0>,
757771 SDTCisVT<1, i32>,
@@ -762,6 +776,7 @@ def AArch64authcall_rvmarker : SDNode<"AArch64ISD::AUTH_CALL_RVMARKER",
762776 [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
763777 SDNPVariadic]>;
764778
779+ // Conditional branch instruction; "b.cond".
765780def AArch64brcond : SDNode<"AArch64ISD::BRCOND", SDT_AArch64Brcond,
766781 [SDNPHasChain]>;
767782def AArch64cbz : SDNode<"AArch64ISD::CBZ", SDT_AArch64cbz,
@@ -775,13 +790,19 @@ def AArch64tbnz : SDNode<"AArch64ISD::TBNZ", SDT_AArch64tbz,
775790
776791
777792def AArch64csel : SDNode<"AArch64ISD::CSEL", SDT_AArch64CSel>;
793+ // Conditional select invert.
778794def AArch64csinv : SDNode<"AArch64ISD::CSINV", SDT_AArch64CSel>;
795+ // Conditional select negate.
779796def AArch64csneg : SDNode<"AArch64ISD::CSNEG", SDT_AArch64CSel>;
797+ // Conditional select increment.
780798def AArch64csinc : SDNode<"AArch64ISD::CSINC", SDT_AArch64CSel>;
799+ // Return with a glue operand. Operand 0 is the chain operand.
781800def AArch64retglue : SDNode<"AArch64ISD::RET_GLUE", SDTNone,
782801 [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>;
783802def AArch64adc : SDNode<"AArch64ISD::ADC", SDTBinaryArithWithFlagsIn >;
784803def AArch64sbc : SDNode<"AArch64ISD::SBC", SDTBinaryArithWithFlagsIn>;
804+
805+ // Arithmetic instructions which write flags.
785806def AArch64add_flag : SDNode<"AArch64ISD::ADDS", SDTBinaryArithWithFlagsOut,
786807 [SDNPCommutative]>;
787808def AArch64sub_flag : SDNode<"AArch64ISD::SUBS", SDTBinaryArithWithFlagsOut>;
@@ -790,15 +811,20 @@ def AArch64and_flag : SDNode<"AArch64ISD::ANDS", SDTBinaryArithWithFlagsOut,
790811def AArch64adc_flag : SDNode<"AArch64ISD::ADCS", SDTBinaryArithWithFlagsInOut>;
791812def AArch64sbc_flag : SDNode<"AArch64ISD::SBCS", SDTBinaryArithWithFlagsInOut>;
792813
814+ // Conditional compares. Operands: left,right,falsecc,cc,flags
793815def AArch64ccmp : SDNode<"AArch64ISD::CCMP", SDT_AArch64CCMP>;
794816def AArch64ccmn : SDNode<"AArch64ISD::CCMN", SDT_AArch64CCMP>;
795817def AArch64fccmp : SDNode<"AArch64ISD::FCCMP", SDT_AArch64FCCMP>;
796818
819+ // Pointer to the thread's local storage area. Materialised from TPIDR_EL0 on
820+ // ELF.
797821def AArch64threadpointer : SDNode<"AArch64ISD::THREAD_POINTER", SDTPtrLeaf>;
798822
823+ // Floating point comparison
799824def AArch64fcmp : SDNode<"AArch64ISD::FCMP", SDT_AArch64FCmp>;
800825
801826let IsStrictFP = true in {
827+ // Strict (exception-raising) floating point comparison
802828 def AArch64strict_fcmp : SDNode<"AArch64ISD::STRICT_FCMP", SDT_AArch64FCmp,
803829 [SDNPHasChain]>;
804830 def AArch64strict_fcmpe : SDNode<"AArch64ISD::STRICT_FCMPE", SDT_AArch64FCmp,
@@ -809,6 +835,7 @@ def AArch64any_fcmp : PatFrags<(ops node:$lhs, node:$rhs),
809835 [(AArch64strict_fcmp node:$lhs, node:$rhs),
810836 (AArch64fcmp node:$lhs, node:$rhs)]>;
811837
838+ // Scalar-to-vector duplication
812839def AArch64dup : SDNode<"AArch64ISD::DUP", SDT_AArch64Dup>;
813840def AArch64duplane8 : SDNode<"AArch64ISD::DUPLANE8", SDT_AArch64DupLane>;
814841def AArch64duplane16 : SDNode<"AArch64ISD::DUPLANE16", SDT_AArch64DupLane>;
@@ -818,13 +845,15 @@ def AArch64duplane128 : SDNode<"AArch64ISD::DUPLANE128", SDT_AArch64DupLane>;
818845
819846def AArch64insr : SDNode<"AArch64ISD::INSR", SDT_AArch64Insr>;
820847
848+ // Vector shuffles
821849def AArch64zip1 : SDNode<"AArch64ISD::ZIP1", SDT_AArch64Zip>;
822850def AArch64zip2 : SDNode<"AArch64ISD::ZIP2", SDT_AArch64Zip>;
823851def AArch64uzp1 : SDNode<"AArch64ISD::UZP1", SDT_AArch64Zip>;
824852def AArch64uzp2 : SDNode<"AArch64ISD::UZP2", SDT_AArch64Zip>;
825853def AArch64trn1 : SDNode<"AArch64ISD::TRN1", SDT_AArch64Zip>;
826854def AArch64trn2 : SDNode<"AArch64ISD::TRN2", SDT_AArch64Zip>;
827855
856+ // Vector immedate moves
828857def AArch64movi_edit : SDNode<"AArch64ISD::MOVIedit", SDT_AArch64MOVIedit>;
829858def AArch64movi_shift : SDNode<"AArch64ISD::MOVIshift", SDT_AArch64MOVIshift>;
830859def AArch64movi_msl : SDNode<"AArch64ISD::MOVImsl", SDT_AArch64MOVIshift>;
@@ -838,23 +867,28 @@ def AArch64rev32 : SDNode<"AArch64ISD::REV32", SDT_AArch64Rev>;
838867def AArch64rev64 : SDNode<"AArch64ISD::REV64", SDT_AArch64Rev>;
839868def AArch64ext : SDNode<"AArch64ISD::EXT", SDT_AArch64ExtVec>;
840869
870+ // Vector shift by scalar
871+ def AArch64vlshr : SDNode<"AArch64ISD::VLSHR", SDT_AArch64vshift>;
872+ def AArch64vshl : SDNode<"AArch64ISD::VSHL", SDT_AArch64vshift>;
841873def AArch64vashr : SDNode<"AArch64ISD::VASHR", SDT_AArch64vshift>;
842874
843875def AArch64vashr_exact : PatFrag<(ops node:$lhs, node:$rhs),
844876 (AArch64vashr node:$lhs, node:$rhs), [{
845877 return N->getFlags().hasExact();
846878}]>;
847879
848- def AArch64vlshr : SDNode<"AArch64ISD::VLSHR", SDT_AArch64vshift>;
849- def AArch64vshl : SDNode<"AArch64ISD::VSHL", SDT_AArch64vshift>;
880+ // Vector shift by scalar (again)
850881def AArch64sqshli : SDNode<"AArch64ISD::SQSHL_I", SDT_AArch64vshift>;
851882def AArch64uqshli : SDNode<"AArch64ISD::UQSHL_I", SDT_AArch64vshift>;
852883def AArch64sqshlui : SDNode<"AArch64ISD::SQSHLU_I", SDT_AArch64vshift>;
853884def AArch64srshri : SDNode<"AArch64ISD::SRSHR_I", SDT_AArch64vshift>;
854885def AArch64urshri : SDNode<"AArch64ISD::URSHR_I", SDT_AArch64vshift>;
886+
855887def AArch64vsli : SDNode<"AArch64ISD::VSLI", SDT_AArch64vshiftinsert>;
856888def AArch64vsri : SDNode<"AArch64ISD::VSRI", SDT_AArch64vshiftinsert>;
857889
890+ // Vector bitwise select: similar to ISD::VSELECT but not all bits within an
891+ // element must be identical.
858892def AArch64bsp: SDNode<"AArch64ISD::BSP", SDT_AArch64trivec>;
859893
860894def AArch64cmeq : PatFrag<(ops node:$lhs, node:$rhs),
@@ -868,6 +902,7 @@ def AArch64cmhi : PatFrag<(ops node:$lhs, node:$rhs),
868902def AArch64cmhs : PatFrag<(ops node:$lhs, node:$rhs),
869903 (setcc node:$lhs, node:$rhs, SETUGE)>;
870904
905+ // Vector comparisons
871906def AArch64fcmeq: SDNode<"AArch64ISD::FCMEQ", SDT_AArch64fcmp>;
872907def AArch64fcmge: SDNode<"AArch64ISD::FCMGE", SDT_AArch64fcmp>;
873908def AArch64fcmgt: SDNode<"AArch64ISD::FCMGT", SDT_AArch64fcmp>;
@@ -902,6 +937,7 @@ def AArch64fcmlez : PatFrag<(ops node:$lhs),
902937def AArch64fcmltz : PatFrag<(ops node:$lhs),
903938 (AArch64fcmgt immAllZerosV, node:$lhs)>;
904939
940+ // Round wide FP to narrow FP with inexact results to odd.
905941def AArch64fcvtxn_n: SDNode<"AArch64ISD::FCVTXN", SDTFPRoundOp>;
906942def AArch64fcvtxnsdr: PatFrags<(ops node:$Rn),
907943 [(f32 (int_aarch64_sisd_fcvtxn (f64 node:$Rn))),
@@ -912,18 +948,24 @@ def AArch64fcvtxnv: PatFrags<(ops node:$Rn),
912948
913949//def Aarch64softf32tobf16v8: SDNode<"AArch64ISD::", SDTFPRoundOp>;
914950
951+ // Vector immediate ops
915952def AArch64bici: SDNode<"AArch64ISD::BICi", SDT_AArch64vecimm>;
916953def AArch64orri: SDNode<"AArch64ISD::ORRi", SDT_AArch64vecimm>;
917954
955+ // Tail calls
918956def AArch64tcret: SDNode<"AArch64ISD::TC_RETURN", SDT_AArch64TCRET,
919957 [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>;
920958
959+ // Custom prefetch handling
921960def AArch64Prefetch : SDNode<"AArch64ISD::PREFETCH", SDT_AArch64PREFETCH,
922961 [SDNPHasChain, SDNPSideEffect]>;
923962
963+ // {s|u}int to FP within a FP register.
924964def AArch64sitof: SDNode<"AArch64ISD::SITOF", SDT_AArch64ITOF>;
925965def AArch64uitof: SDNode<"AArch64ISD::UITOF", SDT_AArch64ITOF>;
926966
967+ // Produces the full sequence of instructions for getting the thread pointer
968+ // offset of a variable into X0, using the TLSDesc model.
927969def AArch64tlsdesc_callseq : SDNode<"AArch64ISD::TLSDESC_CALLSEQ",
928970 SDT_AArch64TLSDescCallSeq,
929971 [SDNPOutGlue, SDNPHasChain, SDNPVariadic]>;
@@ -935,6 +977,11 @@ def AArch64tlsdesc_auth_callseq : SDNode<"AArch64ISD::TLSDESC_AUTH_CALLSEQ",
935977def AArch64WrapperLarge : SDNode<"AArch64ISD::WrapperLarge",
936978 SDT_AArch64WrapperLarge>;
937979
980+ /// Natural vector cast. ISD::BITCAST is not natural in the big-endian
981+ /// world w.r.t vectors; which causes additional REV instructions to be
982+ /// generated to compensate for the byte-swapping. But sometimes we do
983+ /// need to re-interpret the data in SIMD vector registers in big-endian
984+ /// mode without emitting such REV instructions.
938985def AArch64NvCast : SDNode<"AArch64ISD::NVCAST", SDTUnaryOp>;
939986
940987def SDT_AArch64mull : SDTypeProfile<1, 2, [SDTCisInt<0>, SDTCisInt<1>,
@@ -946,21 +993,30 @@ def AArch64smull : SDNode<"AArch64ISD::SMULL", SDT_AArch64mull,
946993def AArch64umull : SDNode<"AArch64ISD::UMULL", SDT_AArch64mull,
947994 [SDNPCommutative]>;
948995
996+ // Reciprocal estimates and steps.
949997def AArch64frecpe : SDNode<"AArch64ISD::FRECPE", SDTFPUnaryOp>;
950998def AArch64frecps : SDNode<"AArch64ISD::FRECPS", SDTFPBinOp>;
951999def AArch64frsqrte : SDNode<"AArch64ISD::FRSQRTE", SDTFPUnaryOp>;
9521000def AArch64frsqrts : SDNode<"AArch64ISD::FRSQRTS", SDTFPBinOp>;
9531001
1002+ // udot/sdot/usdot instructions
9541003def AArch64sdot : SDNode<"AArch64ISD::SDOT", SDT_AArch64Dot>;
9551004def AArch64udot : SDNode<"AArch64ISD::UDOT", SDT_AArch64Dot>;
9561005def AArch64usdot : SDNode<"AArch64ISD::USDOT", SDT_AArch64Dot>;
9571006
1007+ // Vector across-lanes addition
1008+ // Only the lower result lane is defined.
9581009def AArch64saddv : SDNode<"AArch64ISD::SADDV", SDT_AArch64UnaryVec>;
9591010def AArch64uaddv : SDNode<"AArch64ISD::UADDV", SDT_AArch64UnaryVec>;
1011+
1012+ // Vector across-lanes min/max
1013+ // Only the lower result lane is defined.
9601014def AArch64sminv : SDNode<"AArch64ISD::SMINV", SDT_AArch64UnaryVec>;
9611015def AArch64uminv : SDNode<"AArch64ISD::UMINV", SDT_AArch64UnaryVec>;
9621016def AArch64smaxv : SDNode<"AArch64ISD::SMAXV", SDT_AArch64UnaryVec>;
9631017def AArch64umaxv : SDNode<"AArch64ISD::UMAXV", SDT_AArch64UnaryVec>;
1018+
1019+ // Unsigned sum Long across Vector
9641020def AArch64uaddlv : SDNode<"AArch64ISD::UADDLV", SDT_AArch64uaddlp>;
9651021def AArch64saddlv : SDNode<"AArch64ISD::SADDLV", SDT_AArch64uaddlp>;
9661022
@@ -971,7 +1027,9 @@ def AArch64sabd : PatFrags<(ops node:$lhs, node:$rhs),
9711027 [(abds node:$lhs, node:$rhs),
9721028 (int_aarch64_neon_sabd node:$lhs, node:$rhs)]>;
9731029
1030+ // Add Pairwise of two vectors
9741031def AArch64addp_n : SDNode<"AArch64ISD::ADDP", SDT_AArch64Zip>;
1032+ // Add Long Pairwise
9751033def AArch64uaddlp_n : SDNode<"AArch64ISD::UADDLP", SDT_AArch64uaddlp>;
9761034def AArch64saddlp_n : SDNode<"AArch64ISD::SADDLP", SDT_AArch64uaddlp>;
9771035def AArch64addp : PatFrags<(ops node:$Rn, node:$Rm),
@@ -1033,11 +1091,14 @@ def AArch64stnp : SDNode<"AArch64ISD::STNP", SDT_AArch64stnp, [SDNPHasChain, SDN
10331091
10341092def AArch64tbl : SDNode<"AArch64ISD::TBL", SDT_AArch64TBL>;
10351093
1094+ // To avoid stack clash, allocation is performed by block and each block is
1095+ // probed.
10361096def AArch64probedalloca
10371097 : SDNode<"AArch64ISD::PROBED_ALLOCA",
10381098 SDTypeProfile<0, 1, [SDTCisPtrTy<0>]>,
10391099 [SDNPHasChain, SDNPMayStore]>;
10401100
1101+ // MRS, also sets the flags via a glue.
10411102def AArch64mrs : SDNode<"AArch64ISD::MRS",
10421103 SDTypeProfile<2, 1, [SDTCisVT<0, i64>,
10431104 SDTCisVT<1, i32>,
@@ -1058,6 +1119,7 @@ def AArch64msrr : SDNode<"AArch64ISD::MSRR",
10581119 [SDNPHasChain]>;
10591120
10601121def SD_AArch64rshrnb : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisVec<1>, SDTCisInt<2>]>;
1122+ // Vector narrowing shift by immediate (bottom)
10611123def AArch64rshrnb : SDNode<"AArch64ISD::RSHRNB_I", SD_AArch64rshrnb>;
10621124def AArch64rshrnb_pf : PatFrags<(ops node:$rs, node:$i),
10631125 [(AArch64rshrnb node:$rs, node:$i),
@@ -1066,6 +1128,8 @@ def AArch64rshrnb_pf : PatFrags<(ops node:$rs, node:$i),
10661128def AArch64CttzElts : SDNode<"AArch64ISD::CTTZ_ELTS", SDTypeProfile<1, 1,
10671129 [SDTCisInt<0>, SDTCisVec<1>]>, []>;
10681130
1131+ // NEON Load/Store with post-increment base updates.
1132+ // TODO: Complete SDTypeProfile constraints.
10691133def AArch64ld2post : SDNode<"AArch64ISD::LD2post", SDTypeProfile<3, 2, []>, [
10701134 SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
10711135def AArch64ld3post : SDNode<"AArch64ISD::LD3post", SDTypeProfile<4, 2, []>, [
@@ -10345,6 +10409,8 @@ def StoreSwiftAsyncContext
1034510409 : Pseudo<(outs), (ins GPR64:$ctx, GPR64sp:$base, simm9:$offset),
1034610410 []>, Sched<[]>;
1034710411
10412+ // Asserts that a function argument (i32) is zero-extended to i8 by
10413+ // the caller
1034810414def AArch64AssertZExtBool : SDNode<"AArch64ISD::ASSERT_ZEXT_BOOL", SDT_assert>;
1034910415def : Pat<(AArch64AssertZExtBool GPR32:$op),
1035010416 (i32 GPR32:$op)>;
0 commit comments