From 711b56850fd18b6f9dc561f95f6e333d465ab933 Mon Sep 17 00:00:00 2001 From: Kerry McLaughlin Date: Tue, 1 Apr 2025 16:46:56 +0000 Subject: [PATCH 1/5] [AArch64] Allow lowering of more types to GET_ACTIVE_LANE_MASK Adds support for operand promotion and splitting/widening the result of the ISD::GET_ACTIVE_LANE_MASK node. For AArch64, shouldExpandGetActiveLaneMask now returns false for more types which we know can be legalised. --- .../SelectionDAG/LegalizeIntegerTypes.cpp | 10 + llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 3 + .../SelectionDAG/LegalizeVectorTypes.cpp | 27 ++ .../Target/AArch64/AArch64ISelLowering.cpp | 11 +- .../CostModel/AArch64/sve-intrinsics.ll | 24 +- llvm/test/CodeGen/AArch64/active_lane_mask.ll | 279 ++++-------------- 6 files changed, 115 insertions(+), 239 deletions(-) diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp index 9ccd6a4d1684c..65fe4174cc6cc 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -2088,6 +2088,9 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { case ISD::VECTOR_FIND_LAST_ACTIVE: Res = PromoteIntOp_VECTOR_FIND_LAST_ACTIVE(N, OpNo); break; + case ISD::GET_ACTIVE_LANE_MASK: + Res = PromoteIntOp_GET_ACTIVE_LANE_MASK(N); + break; case ISD::PARTIAL_REDUCE_UMLA: case ISD::PARTIAL_REDUCE_SMLA: Res = PromoteIntOp_PARTIAL_REDUCE_MLA(N); @@ -2874,6 +2877,13 @@ SDValue DAGTypeLegalizer::PromoteIntOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N, return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); } +SDValue DAGTypeLegalizer::PromoteIntOp_GET_ACTIVE_LANE_MASK(SDNode *N) { + SmallVector NewOps(N->ops()); + NewOps[0] = ZExtPromotedInteger(N->getOperand(0)); + NewOps[1] = ZExtPromotedInteger(N->getOperand(1)); + return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); +} + SDValue DAGTypeLegalizer::PromoteIntOp_PARTIAL_REDUCE_MLA(SDNode *N) { SmallVector NewOps(N->ops()); if (N->getOpcode() == ISD::PARTIAL_REDUCE_SMLA) { diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h index cf3a9e23f4878..dd9af47da5287 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -432,6 +432,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer { SDValue PromoteIntOp_VP_SPLICE(SDNode *N, unsigned OpNo); SDValue PromoteIntOp_VECTOR_HISTOGRAM(SDNode *N, unsigned OpNo); SDValue PromoteIntOp_VECTOR_FIND_LAST_ACTIVE(SDNode *N, unsigned OpNo); + SDValue PromoteIntOp_GET_ACTIVE_LANE_MASK(SDNode *N); SDValue PromoteIntOp_PARTIAL_REDUCE_MLA(SDNode *N); void SExtOrZExtPromotedOperands(SDValue &LHS, SDValue &RHS); @@ -985,6 +986,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer { void SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo, SDValue &Hi); void SplitVecRes_VP_REVERSE(SDNode *N, SDValue &Lo, SDValue &Hi); void SplitVecRes_PARTIAL_REDUCE_MLA(SDNode *N, SDValue &Lo, SDValue &Hi); + void SplitVecRes_GET_ACTIVE_LANE_MASK(SDNode *N, SDValue &Lo, SDValue &Hi); // Vector Operand Splitting: <128 x ty> -> 2 x <64 x ty>. bool SplitVectorOperand(SDNode *N, unsigned OpNo); @@ -1081,6 +1083,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer { SDValue WidenVecRes_UNDEF(SDNode *N); SDValue WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N); SDValue WidenVecRes_VECTOR_REVERSE(SDNode *N); + SDValue WidenVecRes_GET_ACTIVE_LANE_MASK(SDNode *N); SDValue WidenVecRes_Ternary(SDNode *N); SDValue WidenVecRes_Binary(SDNode *N); diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp index 0c0e700f6abca..b2c941d435451 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -1401,6 +1401,9 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { case ISD::PARTIAL_REDUCE_SMLA: SplitVecRes_PARTIAL_REDUCE_MLA(N, Lo, Hi); break; + case ISD::GET_ACTIVE_LANE_MASK: + SplitVecRes_GET_ACTIVE_LANE_MASK(N, Lo, Hi); + break; } // If Lo/Hi is null, the sub-method took care of registering results etc. @@ -3248,6 +3251,22 @@ void DAGTypeLegalizer::SplitVecRes_PARTIAL_REDUCE_MLA(SDNode *N, SDValue &Lo, Hi = DAG.getNode(Opcode, DL, ResultVT, AccHi, Input1Hi, Input2Hi); } +void DAGTypeLegalizer::SplitVecRes_GET_ACTIVE_LANE_MASK(SDNode *N, SDValue &Lo, + SDValue &Hi) { + SDLoc DL(N); + SDValue Op0 = N->getOperand(0); + SDValue Op1 = N->getOperand(1); + EVT OpVT = Op0.getValueType(); + + EVT LoVT, HiVT; + std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0)); + + Lo = DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, DL, LoVT, Op0, Op1); + SDValue LoElts = DAG.getElementCount(DL, OpVT, LoVT.getVectorElementCount()); + SDValue HiStartVal = DAG.getNode(ISD::UADDSAT, DL, OpVT, Op0, LoElts); + Hi = DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, DL, HiVT, HiStartVal, Op1); +} + void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N) { unsigned Factor = N->getNumOperands(); @@ -4648,6 +4667,9 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { case ISD::VECTOR_REVERSE: Res = WidenVecRes_VECTOR_REVERSE(N); break; + case ISD::GET_ACTIVE_LANE_MASK: + Res = WidenVecRes_GET_ACTIVE_LANE_MASK(N); + break; case ISD::ADD: case ISD::VP_ADD: case ISD::AND: case ISD::VP_AND: @@ -6595,6 +6617,11 @@ SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_REVERSE(SDNode *N) { Mask); } +SDValue DAGTypeLegalizer::WidenVecRes_GET_ACTIVE_LANE_MASK(SDNode *N) { + EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); + return DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, SDLoc(N), NVT, N->ops()); +} + SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) { assert(N->getValueType(0).isVector() && N->getOperand(0).getValueType().isVector() && diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index fb7f7d6f7537d..8135bf02c2711 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -2105,15 +2105,12 @@ bool AArch64TargetLowering::shouldExpandGetActiveLaneMask(EVT ResVT, if (!Subtarget->hasSVE()) return true; - // We can only support legal predicate result types. We can use the SVE - // whilelo instruction for generating fixed-width predicates too. - if (ResVT != MVT::nxv2i1 && ResVT != MVT::nxv4i1 && ResVT != MVT::nxv8i1 && - ResVT != MVT::nxv16i1 && ResVT != MVT::v2i1 && ResVT != MVT::v4i1 && - ResVT != MVT::v8i1 && ResVT != MVT::v16i1) + if (!ResVT.isVector() || ResVT.getVectorElementType() != MVT::i1) return true; - // The whilelo instruction only works with i32 or i64 scalar inputs. - if (OpVT != MVT::i32 && OpVT != MVT::i64) + // 32 & 64 bit operands are supported. We can promote anything < 64 bits, + // but anything larger should be expanded. + if (OpVT.bitsGT(MVT::i64)) return true; return false; diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll index 1716f48f4b042..bcc70f44a2adf 100644 --- a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll +++ b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll @@ -920,8 +920,8 @@ define void @get_lane_mask() #0 { ; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 1 for: %mask_nxv8i1_i32 = call @llvm.get.active.lane.mask.nxv8i1.i32(i32 undef, i32 undef) ; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 1 for: %mask_nxv4i1_i32 = call @llvm.get.active.lane.mask.nxv4i1.i32(i32 undef, i32 undef) ; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 1 for: %mask_nxv2i1_i32 = call @llvm.get.active.lane.mask.nxv2i1.i32(i32 undef, i32 undef) -; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:48 CodeSize:33 Lat:33 SizeLat:33 for: %mask_nxv32i1_i64 = call @llvm.get.active.lane.mask.nxv32i1.i64(i64 undef, i64 undef) -; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:6 CodeSize:5 Lat:5 SizeLat:5 for: %mask_nxv16i1_i16 = call @llvm.get.active.lane.mask.nxv16i1.i16(i16 undef, i16 undef) +; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 2 for: %mask_nxv32i1_i64 = call @llvm.get.active.lane.mask.nxv32i1.i64(i64 undef, i64 undef) +; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 1 for: %mask_nxv16i1_i16 = call @llvm.get.active.lane.mask.nxv16i1.i16(i16 undef, i16 undef) ; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 32 for: %mask_v16i1_i64 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 undef, i64 undef) ; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 16 for: %mask_v8i1_i64 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 undef, i64 undef) ; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 8 for: %mask_v4i1_i64 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 undef, i64 undef) @@ -930,8 +930,8 @@ define void @get_lane_mask() #0 { ; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 16 for: %mask_v8i1_i32 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 undef, i32 undef) ; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 8 for: %mask_v4i1_i32 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 undef, i32 undef) ; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 4 for: %mask_v2i1_i32 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 undef, i32 undef) -; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:48 CodeSize:33 Lat:33 SizeLat:33 for: %mask_v32i1_i64 = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 undef, i64 undef) -; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:6 CodeSize:5 Lat:5 SizeLat:5 for: %mask_v16i1_i16 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16 undef, i16 undef) +; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 64 for: %mask_v32i1_i64 = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 undef, i64 undef) +; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 32 for: %mask_v16i1_i16 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16 undef, i16 undef) ; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; ; CHECK-VSCALE-2-LABEL: 'get_lane_mask' @@ -943,8 +943,8 @@ define void @get_lane_mask() #0 { ; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 1 for: %mask_nxv8i1_i32 = call @llvm.get.active.lane.mask.nxv8i1.i32(i32 undef, i32 undef) ; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 1 for: %mask_nxv4i1_i32 = call @llvm.get.active.lane.mask.nxv4i1.i32(i32 undef, i32 undef) ; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 1 for: %mask_nxv2i1_i32 = call @llvm.get.active.lane.mask.nxv2i1.i32(i32 undef, i32 undef) -; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:48 CodeSize:33 Lat:33 SizeLat:33 for: %mask_nxv32i1_i64 = call @llvm.get.active.lane.mask.nxv32i1.i64(i64 undef, i64 undef) -; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:6 CodeSize:5 Lat:5 SizeLat:5 for: %mask_nxv16i1_i16 = call @llvm.get.active.lane.mask.nxv16i1.i16(i16 undef, i16 undef) +; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 2 for: %mask_nxv32i1_i64 = call @llvm.get.active.lane.mask.nxv32i1.i64(i64 undef, i64 undef) +; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 1 for: %mask_nxv16i1_i16 = call @llvm.get.active.lane.mask.nxv16i1.i16(i16 undef, i16 undef) ; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 32 for: %mask_v16i1_i64 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 undef, i64 undef) ; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 16 for: %mask_v8i1_i64 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 undef, i64 undef) ; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 8 for: %mask_v4i1_i64 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 undef, i64 undef) @@ -953,8 +953,8 @@ define void @get_lane_mask() #0 { ; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 16 for: %mask_v8i1_i32 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 undef, i32 undef) ; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 8 for: %mask_v4i1_i32 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 undef, i32 undef) ; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 4 for: %mask_v2i1_i32 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 undef, i32 undef) -; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:48 CodeSize:33 Lat:33 SizeLat:33 for: %mask_v32i1_i64 = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 undef, i64 undef) -; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:6 CodeSize:5 Lat:5 SizeLat:5 for: %mask_v16i1_i16 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16 undef, i16 undef) +; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 64 for: %mask_v32i1_i64 = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 undef, i64 undef) +; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 32 for: %mask_v16i1_i16 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16 undef, i16 undef) ; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; ; TYPE_BASED_ONLY-LABEL: 'get_lane_mask' @@ -966,8 +966,8 @@ define void @get_lane_mask() #0 { ; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 1 for: %mask_nxv8i1_i32 = call @llvm.get.active.lane.mask.nxv8i1.i32(i32 undef, i32 undef) ; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 1 for: %mask_nxv4i1_i32 = call @llvm.get.active.lane.mask.nxv4i1.i32(i32 undef, i32 undef) ; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 1 for: %mask_nxv2i1_i32 = call @llvm.get.active.lane.mask.nxv2i1.i32(i32 undef, i32 undef) -; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:48 CodeSize:33 Lat:33 SizeLat:33 for: %mask_nxv32i1_i64 = call @llvm.get.active.lane.mask.nxv32i1.i64(i64 undef, i64 undef) -; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:6 CodeSize:5 Lat:5 SizeLat:5 for: %mask_nxv16i1_i16 = call @llvm.get.active.lane.mask.nxv16i1.i16(i16 undef, i16 undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 2 for: %mask_nxv32i1_i64 = call @llvm.get.active.lane.mask.nxv32i1.i64(i64 undef, i64 undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 1 for: %mask_nxv16i1_i16 = call @llvm.get.active.lane.mask.nxv16i1.i16(i16 undef, i16 undef) ; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 32 for: %mask_v16i1_i64 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i64(i64 undef, i64 undef) ; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 16 for: %mask_v8i1_i64 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i64(i64 undef, i64 undef) ; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 8 for: %mask_v4i1_i64 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i64(i64 undef, i64 undef) @@ -976,8 +976,8 @@ define void @get_lane_mask() #0 { ; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 16 for: %mask_v8i1_i32 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 undef, i32 undef) ; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 8 for: %mask_v4i1_i32 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 undef, i32 undef) ; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 4 for: %mask_v2i1_i32 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 undef, i32 undef) -; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:48 CodeSize:33 Lat:33 SizeLat:33 for: %mask_v32i1_i64 = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 undef, i64 undef) -; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:6 CodeSize:5 Lat:5 SizeLat:5 for: %mask_v16i1_i16 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16 undef, i16 undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 64 for: %mask_v32i1_i64 = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 undef, i64 undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 32 for: %mask_v16i1_i16 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16 undef, i16 undef) ; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; %mask_nxv16i1_i64 = call @llvm.get.active.lane.mask.nxv16i1.i64(i64 undef, i64 undef) diff --git a/llvm/test/CodeGen/AArch64/active_lane_mask.ll b/llvm/test/CodeGen/AArch64/active_lane_mask.ll index 025bbf749fc71..f4e0c0aac4495 100644 --- a/llvm/test/CodeGen/AArch64/active_lane_mask.ll +++ b/llvm/test/CodeGen/AArch64/active_lane_mask.ll @@ -78,12 +78,9 @@ define @lane_mask_nxv2i1_i64(i64 %index, i64 %TC) { define @lane_mask_nxv16i1_i8(i8 %index, i8 %TC) { ; CHECK-LABEL: lane_mask_nxv16i1_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: index z0.b, #0, #1 -; CHECK-NEXT: mov z1.b, w0 -; CHECK-NEXT: ptrue p0.b -; CHECK-NEXT: uqadd z0.b, z0.b, z1.b -; CHECK-NEXT: mov z1.b, w1 -; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b +; CHECK-NEXT: and w8, w1, #0xff +; CHECK-NEXT: and w9, w0, #0xff +; CHECK-NEXT: whilelo p0.b, w9, w8 ; CHECK-NEXT: ret %active.lane.mask = call @llvm.get.active.lane.mask.nxv16i1.i8(i8 %index, i8 %TC) ret %active.lane.mask @@ -92,16 +89,9 @@ define @lane_mask_nxv16i1_i8(i8 %index, i8 %TC) { define @lane_mask_nxv8i1_i8(i8 %index, i8 %TC) { ; CHECK-LABEL: lane_mask_nxv8i1_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: index z0.h, #0, #1 -; CHECK-NEXT: mov z1.h, w0 -; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: and z1.h, z1.h, #0xff -; CHECK-NEXT: and z0.h, z0.h, #0xff -; CHECK-NEXT: add z0.h, z0.h, z1.h -; CHECK-NEXT: mov z1.h, w1 -; CHECK-NEXT: umin z0.h, z0.h, #255 -; CHECK-NEXT: and z1.h, z1.h, #0xff -; CHECK-NEXT: cmphi p0.h, p0/z, z1.h, z0.h +; CHECK-NEXT: and w8, w1, #0xff +; CHECK-NEXT: and w9, w0, #0xff +; CHECK-NEXT: whilelo p0.h, w9, w8 ; CHECK-NEXT: ret %active.lane.mask = call @llvm.get.active.lane.mask.nxv8i1.i8(i8 %index, i8 %TC) ret %active.lane.mask @@ -110,16 +100,9 @@ define @lane_mask_nxv8i1_i8(i8 %index, i8 %TC) { define @lane_mask_nxv4i1_i8(i8 %index, i8 %TC) { ; CHECK-LABEL: lane_mask_nxv4i1_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: index z0.s, #0, #1 -; CHECK-NEXT: and w8, w0, #0xff -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: mov z1.s, w8 ; CHECK-NEXT: and w8, w1, #0xff -; CHECK-NEXT: and z0.s, z0.s, #0xff -; CHECK-NEXT: add z0.s, z0.s, z1.s -; CHECK-NEXT: mov z1.s, w8 -; CHECK-NEXT: umin z0.s, z0.s, #255 -; CHECK-NEXT: cmphi p0.s, p0/z, z1.s, z0.s +; CHECK-NEXT: and w9, w0, #0xff +; CHECK-NEXT: whilelo p0.s, w9, w8 ; CHECK-NEXT: ret %active.lane.mask = call @llvm.get.active.lane.mask.nxv4i1.i8(i8 %index, i8 %TC) ret %active.lane.mask @@ -128,18 +111,9 @@ define @lane_mask_nxv4i1_i8(i8 %index, i8 %TC) { define @lane_mask_nxv2i1_i8(i8 %index, i8 %TC) { ; CHECK-LABEL: lane_mask_nxv2i1_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: index z0.d, #0, #1 -; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 -; CHECK-NEXT: and x8, x0, #0xff -; CHECK-NEXT: // kill: def $w1 killed $w1 def $x1 -; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: mov z1.d, x8 -; CHECK-NEXT: and x8, x1, #0xff -; CHECK-NEXT: and z0.d, z0.d, #0xff -; CHECK-NEXT: add z0.d, z0.d, z1.d -; CHECK-NEXT: mov z1.d, x8 -; CHECK-NEXT: umin z0.d, z0.d, #255 -; CHECK-NEXT: cmphi p0.d, p0/z, z1.d, z0.d +; CHECK-NEXT: and w8, w1, #0xff +; CHECK-NEXT: and w9, w0, #0xff +; CHECK-NEXT: whilelo p0.d, w9, w8 ; CHECK-NEXT: ret %active.lane.mask = call @llvm.get.active.lane.mask.nxv2i1.i8(i8 %index, i8 %TC) ret %active.lane.mask @@ -151,56 +125,11 @@ define @lane_mask_nxv2i1_i8(i8 %index, i8 %TC) { define @lane_mask_nxv32i1_i32(i32 %index, i32 %TC) { ; CHECK-LABEL: lane_mask_nxv32i1_i32: ; CHECK: // %bb.0: -; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill -; CHECK-NEXT: addvl sp, sp, #-1 -; CHECK-NEXT: str p7, [sp, #4, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: str p6, [sp, #5, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: str p5, [sp, #6, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x08, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 8 * VG -; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: index z0.s, #0, #1 -; CHECK-NEXT: mov z1.s, w0 -; CHECK-NEXT: mov z25.s, w1 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: mov z2.d, z0.d -; CHECK-NEXT: mov z3.d, z0.d -; CHECK-NEXT: uqadd z6.s, z0.s, z1.s -; CHECK-NEXT: incw z0.s, all, mul #4 -; CHECK-NEXT: incw z2.s -; CHECK-NEXT: incw z3.s, all, mul #2 -; CHECK-NEXT: uqadd z0.s, z0.s, z1.s -; CHECK-NEXT: cmphi p2.s, p0/z, z25.s, z6.s -; CHECK-NEXT: mov z4.d, z2.d -; CHECK-NEXT: uqadd z5.s, z2.s, z1.s -; CHECK-NEXT: uqadd z7.s, z3.s, z1.s -; CHECK-NEXT: incw z2.s, all, mul #4 -; CHECK-NEXT: incw z3.s, all, mul #4 -; CHECK-NEXT: cmphi p5.s, p0/z, z25.s, z0.s -; CHECK-NEXT: incw z4.s, all, mul #2 -; CHECK-NEXT: uqadd z2.s, z2.s, z1.s -; CHECK-NEXT: uqadd z3.s, z3.s, z1.s -; CHECK-NEXT: cmphi p1.s, p0/z, z25.s, z5.s -; CHECK-NEXT: cmphi p3.s, p0/z, z25.s, z7.s -; CHECK-NEXT: uqadd z24.s, z4.s, z1.s -; CHECK-NEXT: incw z4.s, all, mul #4 -; CHECK-NEXT: cmphi p6.s, p0/z, z25.s, z2.s -; CHECK-NEXT: cmphi p7.s, p0/z, z25.s, z3.s -; CHECK-NEXT: uzp1 p1.h, p2.h, p1.h -; CHECK-NEXT: uqadd z1.s, z4.s, z1.s -; CHECK-NEXT: cmphi p4.s, p0/z, z25.s, z24.s -; CHECK-NEXT: cmphi p0.s, p0/z, z25.s, z1.s -; CHECK-NEXT: uzp1 p2.h, p3.h, p4.h -; CHECK-NEXT: uzp1 p3.h, p5.h, p6.h -; CHECK-NEXT: ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p4.h, p7.h, p0.h -; CHECK-NEXT: ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p0.b, p1.b, p2.b -; CHECK-NEXT: uzp1 p1.b, p3.b, p4.b -; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: addvl sp, sp, #1 -; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: rdvl x8, #1 +; CHECK-NEXT: adds w8, w0, w8 +; CHECK-NEXT: csinv w8, w8, wzr, lo +; CHECK-NEXT: whilelo p0.b, w0, w1 +; CHECK-NEXT: whilelo p1.b, w8, w1 ; CHECK-NEXT: ret %active.lane.mask = call @llvm.get.active.lane.mask.nxv32i1.i32(i32 %index, i32 %TC) ret %active.lane.mask @@ -209,99 +138,11 @@ define @lane_mask_nxv32i1_i32(i32 %index, i32 %TC) { define @lane_mask_nxv32i1_i64(i64 %index, i64 %TC) { ; CHECK-LABEL: lane_mask_nxv32i1_i64: ; CHECK: // %bb.0: -; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill -; CHECK-NEXT: addvl sp, sp, #-2 -; CHECK-NEXT: str p9, [sp, #2, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: str p8, [sp, #3, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: str p7, [sp, #4, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: str p6, [sp, #5, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: str p5, [sp, #6, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: str p4, [sp, #7, mul vl] // 2-byte Folded Spill -; CHECK-NEXT: str z8, [sp, #1, mul vl] // 16-byte Folded Spill -; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG -; CHECK-NEXT: .cfi_offset w29, -16 -; CHECK-NEXT: .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x70, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 16 - 8 * VG -; CHECK-NEXT: index z5.d, #0, #1 -; CHECK-NEXT: mov z0.d, x0 -; CHECK-NEXT: mov z3.d, x1 -; CHECK-NEXT: ptrue p0.d -; CHECK-NEXT: mov z2.d, z5.d -; CHECK-NEXT: mov z1.d, z5.d -; CHECK-NEXT: mov z4.d, z5.d -; CHECK-NEXT: uqadd z25.d, z5.d, z0.d -; CHECK-NEXT: incd z5.d, all, mul #8 -; CHECK-NEXT: incd z2.d -; CHECK-NEXT: incd z1.d, all, mul #2 -; CHECK-NEXT: incd z4.d, all, mul #4 -; CHECK-NEXT: uqadd z5.d, z5.d, z0.d -; CHECK-NEXT: cmphi p3.d, p0/z, z3.d, z25.d -; CHECK-NEXT: mov z6.d, z2.d -; CHECK-NEXT: mov z7.d, z2.d -; CHECK-NEXT: mov z24.d, z1.d -; CHECK-NEXT: uqadd z26.d, z2.d, z0.d -; CHECK-NEXT: uqadd z27.d, z1.d, z0.d -; CHECK-NEXT: uqadd z28.d, z4.d, z0.d -; CHECK-NEXT: incd z2.d, all, mul #8 -; CHECK-NEXT: incd z1.d, all, mul #8 -; CHECK-NEXT: incd z4.d, all, mul #8 -; CHECK-NEXT: incd z6.d, all, mul #2 -; CHECK-NEXT: incd z7.d, all, mul #4 -; CHECK-NEXT: incd z24.d, all, mul #4 -; CHECK-NEXT: cmphi p4.d, p0/z, z3.d, z26.d -; CHECK-NEXT: cmphi p2.d, p0/z, z3.d, z27.d -; CHECK-NEXT: cmphi p1.d, p0/z, z3.d, z28.d -; CHECK-NEXT: mov z31.d, z6.d -; CHECK-NEXT: uqadd z29.d, z6.d, z0.d -; CHECK-NEXT: uqadd z30.d, z7.d, z0.d -; CHECK-NEXT: uqadd z8.d, z24.d, z0.d -; CHECK-NEXT: incd z6.d, all, mul #8 -; CHECK-NEXT: incd z7.d, all, mul #8 -; CHECK-NEXT: incd z24.d, all, mul #8 -; CHECK-NEXT: uqadd z2.d, z2.d, z0.d -; CHECK-NEXT: uqadd z1.d, z1.d, z0.d -; CHECK-NEXT: incd z31.d, all, mul #4 -; CHECK-NEXT: uqadd z4.d, z4.d, z0.d -; CHECK-NEXT: uzp1 p3.s, p3.s, p4.s -; CHECK-NEXT: cmphi p5.d, p0/z, z3.d, z29.d -; CHECK-NEXT: cmphi p7.d, p0/z, z3.d, z30.d -; CHECK-NEXT: uqadd z6.d, z6.d, z0.d -; CHECK-NEXT: cmphi p6.d, p0/z, z3.d, z8.d -; CHECK-NEXT: ldr z8, [sp, #1, mul vl] // 16-byte Folded Reload -; CHECK-NEXT: uqadd z7.d, z7.d, z0.d -; CHECK-NEXT: uqadd z25.d, z31.d, z0.d -; CHECK-NEXT: incd z31.d, all, mul #8 -; CHECK-NEXT: uqadd z24.d, z24.d, z0.d -; CHECK-NEXT: cmphi p4.d, p0/z, z3.d, z5.d -; CHECK-NEXT: uzp1 p2.s, p2.s, p5.s -; CHECK-NEXT: cmphi p5.d, p0/z, z3.d, z2.d -; CHECK-NEXT: cmphi p9.d, p0/z, z3.d, z6.d -; CHECK-NEXT: uqadd z0.d, z31.d, z0.d -; CHECK-NEXT: uzp1 p1.s, p1.s, p7.s -; CHECK-NEXT: cmphi p7.d, p0/z, z3.d, z1.d -; CHECK-NEXT: cmphi p8.d, p0/z, z3.d, z25.d -; CHECK-NEXT: uzp1 p2.h, p3.h, p2.h -; CHECK-NEXT: cmphi p3.d, p0/z, z3.d, z7.d -; CHECK-NEXT: uzp1 p4.s, p4.s, p5.s -; CHECK-NEXT: uzp1 p5.s, p7.s, p9.s -; CHECK-NEXT: ldr p9, [sp, #2, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p6.s, p6.s, p8.s -; CHECK-NEXT: cmphi p8.d, p0/z, z3.d, z4.d -; CHECK-NEXT: ldr p7, [sp, #4, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p4.h, p4.h, p5.h -; CHECK-NEXT: ldr p5, [sp, #6, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p1.h, p1.h, p6.h -; CHECK-NEXT: cmphi p6.d, p0/z, z3.d, z24.d -; CHECK-NEXT: cmphi p0.d, p0/z, z3.d, z0.d -; CHECK-NEXT: uzp1 p3.s, p8.s, p3.s -; CHECK-NEXT: ldr p8, [sp, #3, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p0.s, p6.s, p0.s -; CHECK-NEXT: ldr p6, [sp, #5, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: uzp1 p3.h, p3.h, p0.h -; CHECK-NEXT: uzp1 p0.b, p2.b, p1.b -; CHECK-NEXT: uzp1 p1.b, p4.b, p3.b -; CHECK-NEXT: ldr p4, [sp, #7, mul vl] // 2-byte Folded Reload -; CHECK-NEXT: addvl sp, sp, #2 -; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: rdvl x8, #1 +; CHECK-NEXT: adds x8, x0, x8 +; CHECK-NEXT: csinv x8, x8, xzr, lo +; CHECK-NEXT: whilelo p0.b, x0, x1 +; CHECK-NEXT: whilelo p1.b, x8, x1 ; CHECK-NEXT: ret %active.lane.mask = call @llvm.get.active.lane.mask.nxv32i1.i64(i64 %index, i64 %TC) ret %active.lane.mask @@ -310,22 +151,29 @@ define @lane_mask_nxv32i1_i64(i64 %index, i64 %TC) { define @lane_mask_nxv32i1_i8(i8 %index, i8 %TC) { ; CHECK-LABEL: lane_mask_nxv32i1_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: index z0.b, #0, #1 ; CHECK-NEXT: rdvl x8, #1 -; CHECK-NEXT: mov z2.b, w0 -; CHECK-NEXT: mov z1.b, w8 -; CHECK-NEXT: ptrue p1.b -; CHECK-NEXT: add z1.b, z0.b, z1.b -; CHECK-NEXT: uqadd z0.b, z0.b, z2.b -; CHECK-NEXT: uqadd z1.b, z1.b, z2.b -; CHECK-NEXT: mov z2.b, w1 -; CHECK-NEXT: cmphi p0.b, p1/z, z2.b, z0.b -; CHECK-NEXT: cmphi p1.b, p1/z, z2.b, z1.b +; CHECK-NEXT: and w9, w0, #0xff +; CHECK-NEXT: mov w10, #255 // =0xff +; CHECK-NEXT: add w8, w9, w8, uxtb +; CHECK-NEXT: and w11, w1, #0xff +; CHECK-NEXT: cmp w8, #255 +; CHECK-NEXT: csel w8, w8, w10, lo +; CHECK-NEXT: whilelo p0.b, w9, w11 +; CHECK-NEXT: whilelo p1.b, w8, w11 ; CHECK-NEXT: ret %active.lane.mask = call @llvm.get.active.lane.mask.nxv32i1.i8(i8 %index, i8 %TC) ret %active.lane.mask } +define @lane_mask_nxv7i1_i64(i64 %index, i64 %TC) { +; CHECK-LABEL: lane_mask_nxv7i1_i64: +; CHECK: // %bb.0: +; CHECK-NEXT: whilelo p0.h, x0, x1 +; CHECK-NEXT: ret + %active.lane.mask = call @llvm.get.active.lane.mask.nxv7i1.i64(i64 %index, i64 %TC) + ret %active.lane.mask +} + ; UTC_ARGS: --disable ; This test exists to protect against a compiler crash caused by an attempt to ; convert (via changeVectorElementType) an MVT into an EVT, which is impossible. @@ -430,11 +278,11 @@ define <2 x i1> @lane_mask_v2i1_i64(i64 %index, i64 %TC) { define <16 x i1> @lane_mask_v16i1_i8(i8 %index, i8 %TC) { ; CHECK-LABEL: lane_mask_v16i1_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: index z0.b, #0, #1 -; CHECK-NEXT: dup v1.16b, w0 -; CHECK-NEXT: uqadd v0.16b, v1.16b, v0.16b -; CHECK-NEXT: dup v1.16b, w1 -; CHECK-NEXT: cmhi v0.16b, v1.16b, v0.16b +; CHECK-NEXT: and w8, w1, #0xff +; CHECK-NEXT: and w9, w0, #0xff +; CHECK-NEXT: whilelo p0.b, w9, w8 +; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff +; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 ; CHECK-NEXT: ret %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i8(i8 %index, i8 %TC) ret <16 x i1> %active.lane.mask @@ -443,11 +291,11 @@ define <16 x i1> @lane_mask_v16i1_i8(i8 %index, i8 %TC) { define <8 x i1> @lane_mask_v8i1_i8(i8 %index, i8 %TC) { ; CHECK-LABEL: lane_mask_v8i1_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: index z0.b, #0, #1 -; CHECK-NEXT: dup v1.8b, w0 -; CHECK-NEXT: uqadd v0.8b, v1.8b, v0.8b -; CHECK-NEXT: dup v1.8b, w1 -; CHECK-NEXT: cmhi v0.8b, v1.8b, v0.8b +; CHECK-NEXT: and w8, w1, #0xff +; CHECK-NEXT: and w9, w0, #0xff +; CHECK-NEXT: whilelo p0.b, w9, w8 +; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 ; CHECK-NEXT: ret %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i8(i8 %index, i8 %TC) ret <8 x i1> %active.lane.mask @@ -456,15 +304,11 @@ define <8 x i1> @lane_mask_v8i1_i8(i8 %index, i8 %TC) { define <4 x i1> @lane_mask_v4i1_i8(i8 %index, i8 %TC) { ; CHECK-LABEL: lane_mask_v4i1_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: dup v0.4h, w0 -; CHECK-NEXT: index z1.h, #0, #1 -; CHECK-NEXT: movi d2, #0xff00ff00ff00ff -; CHECK-NEXT: dup v3.4h, w1 -; CHECK-NEXT: bic v0.4h, #255, lsl #8 -; CHECK-NEXT: bic v3.4h, #255, lsl #8 -; CHECK-NEXT: add v0.4h, v0.4h, v1.4h -; CHECK-NEXT: umin v0.4h, v0.4h, v2.4h -; CHECK-NEXT: cmhi v0.4h, v3.4h, v0.4h +; CHECK-NEXT: and w8, w1, #0xff +; CHECK-NEXT: and w9, w0, #0xff +; CHECK-NEXT: whilelo p0.h, w9, w8 +; CHECK-NEXT: mov z0.h, p0/z, #-1 // =0xffffffffffffffff +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 ; CHECK-NEXT: ret %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i8(i8 %index, i8 %TC) ret <4 x i1> %active.lane.mask @@ -473,15 +317,11 @@ define <4 x i1> @lane_mask_v4i1_i8(i8 %index, i8 %TC) { define <2 x i1> @lane_mask_v2i1_i8(i8 %index, i8 %TC) { ; CHECK-LABEL: lane_mask_v2i1_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: movi d0, #0x0000ff000000ff -; CHECK-NEXT: dup v1.2s, w0 -; CHECK-NEXT: index z2.s, #0, #1 -; CHECK-NEXT: dup v3.2s, w1 -; CHECK-NEXT: and v1.8b, v1.8b, v0.8b -; CHECK-NEXT: add v1.2s, v1.2s, v2.2s -; CHECK-NEXT: and v2.8b, v3.8b, v0.8b -; CHECK-NEXT: umin v0.2s, v1.2s, v0.2s -; CHECK-NEXT: cmhi v0.2s, v2.2s, v0.2s +; CHECK-NEXT: and w8, w1, #0xff +; CHECK-NEXT: and w9, w0, #0xff +; CHECK-NEXT: whilelo p0.s, w9, w8 +; CHECK-NEXT: mov z0.s, p0/z, #-1 // =0xffffffffffffffff +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 ; CHECK-NEXT: ret %active.lane.mask = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i8(i8 %index, i8 %TC) ret <2 x i1> %active.lane.mask @@ -539,7 +379,6 @@ entry: ret %active.lane.mask } - declare @llvm.get.active.lane.mask.nxv32i1.i32(i32, i32) declare @llvm.get.active.lane.mask.nxv16i1.i32(i32, i32) declare @llvm.get.active.lane.mask.nxv8i1.i32(i32, i32) @@ -556,10 +395,10 @@ declare @llvm.get.active.lane.mask.nxv2i1.i64(i64, i64) declare @llvm.get.active.lane.mask.nxv32i1.i8(i8, i8) declare @llvm.get.active.lane.mask.nxv16i1.i8(i8, i8) declare @llvm.get.active.lane.mask.nxv8i1.i8(i8, i8) +declare @llvm.get.active.lane.mask.nxv7i1.i8(i8, i8) declare @llvm.get.active.lane.mask.nxv4i1.i8(i8, i8) declare @llvm.get.active.lane.mask.nxv2i1.i8(i8, i8) - declare <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32, i32) declare <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32, i32) declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32) From c6700094cb3294c9ad7e1650c24f7f541e84cb20 Mon Sep 17 00:00:00 2001 From: Kerry McLaughlin Date: Tue, 20 May 2025 12:41:34 +0000 Subject: [PATCH 2/5] - Remove !ResVT.isVector() from shouldExpandGetActiveLaneMask --- llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 8135bf02c2711..890c19b26b398 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -2102,15 +2102,12 @@ void AArch64TargetLowering::addTypeForNEON(MVT VT) { bool AArch64TargetLowering::shouldExpandGetActiveLaneMask(EVT ResVT, EVT OpVT) const { // Only SVE has a 1:1 mapping from intrinsic -> instruction (whilelo). - if (!Subtarget->hasSVE()) - return true; - - if (!ResVT.isVector() || ResVT.getVectorElementType() != MVT::i1) + if (!Subtarget->hasSVE() || ResVT.getVectorElementType() != MVT::i1) return true; // 32 & 64 bit operands are supported. We can promote anything < 64 bits, // but anything larger should be expanded. - if (OpVT.bitsGT(MVT::i64)) + if (OpVT.getFixedSizeInBits() > 64) return true; return false; From c4a7e8ae324dbf687831e9bcbae8f08633323a63 Mon Sep 17 00:00:00 2001 From: Kerry McLaughlin Date: Wed, 21 May 2025 10:57:30 +0000 Subject: [PATCH 3/5] - Expand fixed-length get.active.lane.masks if the result or operands would not be legal when lowering with SVE. --- .../Target/AArch64/AArch64ISelLowering.cpp | 5 ++ .../CostModel/AArch64/sve-intrinsics.ll | 12 ++--- llvm/test/CodeGen/AArch64/active_lane_mask.ll | 48 +++++++++++-------- 3 files changed, 39 insertions(+), 26 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 890c19b26b398..098c3681d9703 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -2105,6 +2105,11 @@ bool AArch64TargetLowering::shouldExpandGetActiveLaneMask(EVT ResVT, if (!Subtarget->hasSVE() || ResVT.getVectorElementType() != MVT::i1) return true; + // Only support illegal types if the result is scalable. + if (ResVT.isFixedLengthVector() && (ResVT.getVectorNumElements() > 16 || + (OpVT != MVT::i32 && OpVT != MVT::i64))) + return true; + // 32 & 64 bit operands are supported. We can promote anything < 64 bits, // but anything larger should be expanded. if (OpVT.getFixedSizeInBits() > 64) diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll index bcc70f44a2adf..ec84c58bf9681 100644 --- a/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll +++ b/llvm/test/Analysis/CostModel/AArch64/sve-intrinsics.ll @@ -930,8 +930,8 @@ define void @get_lane_mask() #0 { ; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 16 for: %mask_v8i1_i32 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 undef, i32 undef) ; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 8 for: %mask_v4i1_i32 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 undef, i32 undef) ; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 4 for: %mask_v2i1_i32 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 undef, i32 undef) -; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 64 for: %mask_v32i1_i64 = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 undef, i64 undef) -; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of 32 for: %mask_v16i1_i16 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16 undef, i16 undef) +; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:48 CodeSize:33 Lat:33 SizeLat:33 for: %mask_v32i1_i64 = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 undef, i64 undef) +; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:6 CodeSize:5 Lat:5 SizeLat:5 for: %mask_v16i1_i16 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16 undef, i16 undef) ; CHECK-VSCALE-1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; ; CHECK-VSCALE-2-LABEL: 'get_lane_mask' @@ -953,8 +953,8 @@ define void @get_lane_mask() #0 { ; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 16 for: %mask_v8i1_i32 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 undef, i32 undef) ; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 8 for: %mask_v4i1_i32 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 undef, i32 undef) ; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 4 for: %mask_v2i1_i32 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 undef, i32 undef) -; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 64 for: %mask_v32i1_i64 = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 undef, i64 undef) -; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of 32 for: %mask_v16i1_i16 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16 undef, i16 undef) +; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:48 CodeSize:33 Lat:33 SizeLat:33 for: %mask_v32i1_i64 = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 undef, i64 undef) +; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:6 CodeSize:5 Lat:5 SizeLat:5 for: %mask_v16i1_i16 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16 undef, i16 undef) ; CHECK-VSCALE-2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; ; TYPE_BASED_ONLY-LABEL: 'get_lane_mask' @@ -976,8 +976,8 @@ define void @get_lane_mask() #0 { ; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 16 for: %mask_v8i1_i32 = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i32(i32 undef, i32 undef) ; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 8 for: %mask_v4i1_i32 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 undef, i32 undef) ; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 4 for: %mask_v2i1_i32 = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i32(i32 undef, i32 undef) -; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 64 for: %mask_v32i1_i64 = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 undef, i64 undef) -; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of 32 for: %mask_v16i1_i16 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16 undef, i16 undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:48 CodeSize:33 Lat:33 SizeLat:33 for: %mask_v32i1_i64 = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 undef, i64 undef) +; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:6 CodeSize:5 Lat:5 SizeLat:5 for: %mask_v16i1_i16 = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i16(i16 undef, i16 undef) ; TYPE_BASED_ONLY-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void ; %mask_nxv16i1_i64 = call @llvm.get.active.lane.mask.nxv16i1.i64(i64 undef, i64 undef) diff --git a/llvm/test/CodeGen/AArch64/active_lane_mask.ll b/llvm/test/CodeGen/AArch64/active_lane_mask.ll index f4e0c0aac4495..b3d0667218a1c 100644 --- a/llvm/test/CodeGen/AArch64/active_lane_mask.ll +++ b/llvm/test/CodeGen/AArch64/active_lane_mask.ll @@ -278,11 +278,11 @@ define <2 x i1> @lane_mask_v2i1_i64(i64 %index, i64 %TC) { define <16 x i1> @lane_mask_v16i1_i8(i8 %index, i8 %TC) { ; CHECK-LABEL: lane_mask_v16i1_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: and w8, w1, #0xff -; CHECK-NEXT: and w9, w0, #0xff -; CHECK-NEXT: whilelo p0.b, w9, w8 -; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0 +; CHECK-NEXT: index z0.b, #0, #1 +; CHECK-NEXT: dup v1.16b, w0 +; CHECK-NEXT: uqadd v0.16b, v1.16b, v0.16b +; CHECK-NEXT: dup v1.16b, w1 +; CHECK-NEXT: cmhi v0.16b, v1.16b, v0.16b ; CHECK-NEXT: ret %active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i8(i8 %index, i8 %TC) ret <16 x i1> %active.lane.mask @@ -291,11 +291,11 @@ define <16 x i1> @lane_mask_v16i1_i8(i8 %index, i8 %TC) { define <8 x i1> @lane_mask_v8i1_i8(i8 %index, i8 %TC) { ; CHECK-LABEL: lane_mask_v8i1_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: and w8, w1, #0xff -; CHECK-NEXT: and w9, w0, #0xff -; CHECK-NEXT: whilelo p0.b, w9, w8 -; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: index z0.b, #0, #1 +; CHECK-NEXT: dup v1.8b, w0 +; CHECK-NEXT: uqadd v0.8b, v1.8b, v0.8b +; CHECK-NEXT: dup v1.8b, w1 +; CHECK-NEXT: cmhi v0.8b, v1.8b, v0.8b ; CHECK-NEXT: ret %active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i8(i8 %index, i8 %TC) ret <8 x i1> %active.lane.mask @@ -304,11 +304,15 @@ define <8 x i1> @lane_mask_v8i1_i8(i8 %index, i8 %TC) { define <4 x i1> @lane_mask_v4i1_i8(i8 %index, i8 %TC) { ; CHECK-LABEL: lane_mask_v4i1_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: and w8, w1, #0xff -; CHECK-NEXT: and w9, w0, #0xff -; CHECK-NEXT: whilelo p0.h, w9, w8 -; CHECK-NEXT: mov z0.h, p0/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: dup v0.4h, w0 +; CHECK-NEXT: index z1.h, #0, #1 +; CHECK-NEXT: movi d2, #0xff00ff00ff00ff +; CHECK-NEXT: dup v3.4h, w1 +; CHECK-NEXT: bic v0.4h, #255, lsl #8 +; CHECK-NEXT: bic v3.4h, #255, lsl #8 +; CHECK-NEXT: add v0.4h, v0.4h, v1.4h +; CHECK-NEXT: umin v0.4h, v0.4h, v2.4h +; CHECK-NEXT: cmhi v0.4h, v3.4h, v0.4h ; CHECK-NEXT: ret %active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i8(i8 %index, i8 %TC) ret <4 x i1> %active.lane.mask @@ -317,11 +321,15 @@ define <4 x i1> @lane_mask_v4i1_i8(i8 %index, i8 %TC) { define <2 x i1> @lane_mask_v2i1_i8(i8 %index, i8 %TC) { ; CHECK-LABEL: lane_mask_v2i1_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: and w8, w1, #0xff -; CHECK-NEXT: and w9, w0, #0xff -; CHECK-NEXT: whilelo p0.s, w9, w8 -; CHECK-NEXT: mov z0.s, p0/z, #-1 // =0xffffffffffffffff -; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0 +; CHECK-NEXT: movi d0, #0x0000ff000000ff +; CHECK-NEXT: dup v1.2s, w0 +; CHECK-NEXT: index z2.s, #0, #1 +; CHECK-NEXT: dup v3.2s, w1 +; CHECK-NEXT: and v1.8b, v1.8b, v0.8b +; CHECK-NEXT: add v1.2s, v1.2s, v2.2s +; CHECK-NEXT: and v2.8b, v3.8b, v0.8b +; CHECK-NEXT: umin v0.2s, v1.2s, v0.2s +; CHECK-NEXT: cmhi v0.2s, v2.2s, v0.2s ; CHECK-NEXT: ret %active.lane.mask = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i8(i8 %index, i8 %TC) ret <2 x i1> %active.lane.mask From 03f752f16223f660877588a52576d226782542ea Mon Sep 17 00:00:00 2001 From: Kerry McLaughlin Date: Thu, 22 May 2025 09:47:22 +0000 Subject: [PATCH 4/5] - Expand get_active_lane_mask with type nxv1i1. --- llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 7 ++++--- llvm/test/CodeGen/AArch64/active_lane_mask.ll | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 098c3681d9703..738717d60f08e 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -2105,9 +2105,10 @@ bool AArch64TargetLowering::shouldExpandGetActiveLaneMask(EVT ResVT, if (!Subtarget->hasSVE() || ResVT.getVectorElementType() != MVT::i1) return true; - // Only support illegal types if the result is scalable. - if (ResVT.isFixedLengthVector() && (ResVT.getVectorNumElements() > 16 || - (OpVT != MVT::i32 && OpVT != MVT::i64))) + // Only support illegal types if the result is scalable and min elements > 1. + if (ResVT.getVectorMinNumElements() == 1 || + (ResVT.isFixedLengthVector() && (ResVT.getVectorNumElements() > 16 || + (OpVT != MVT::i32 && OpVT != MVT::i64)))) return true; // 32 & 64 bit operands are supported. We can promote anything < 64 bits, diff --git a/llvm/test/CodeGen/AArch64/active_lane_mask.ll b/llvm/test/CodeGen/AArch64/active_lane_mask.ll index b3d0667218a1c..c5e7d1265ef25 100644 --- a/llvm/test/CodeGen/AArch64/active_lane_mask.ll +++ b/llvm/test/CodeGen/AArch64/active_lane_mask.ll @@ -174,6 +174,22 @@ define @lane_mask_nxv7i1_i64(i64 %index, i64 %TC) { ret %active.lane.mask } +define @lane_mask_nxv1i1_i8(i32 %index, i32 %TC) { +; CHECK-LABEL: lane_mask_nxv1i1_i8: +; CHECK: // %bb.0: +; CHECK-NEXT: index z0.s, #0, #1 +; CHECK-NEXT: mov z1.s, w0 +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: uqadd z0.s, z0.s, z1.s +; CHECK-NEXT: mov z1.s, w1 +; CHECK-NEXT: cmphi p0.s, p0/z, z1.s, z0.s +; CHECK-NEXT: punpklo p0.h, p0.b +; CHECK-NEXT: punpklo p0.h, p0.b +; CHECK-NEXT: ret + %active.lane.mask = call @llvm.get.active.lane.mask.nxv1i1.i32(i32 %index, i32 %TC) + ret %active.lane.mask +} + ; UTC_ARGS: --disable ; This test exists to protect against a compiler crash caused by an attempt to ; convert (via changeVectorElementType) an MVT into an EVT, which is impossible. @@ -392,6 +408,7 @@ declare @llvm.get.active.lane.mask.nxv16i1.i32(i32, i32) declare @llvm.get.active.lane.mask.nxv8i1.i32(i32, i32) declare @llvm.get.active.lane.mask.nxv4i1.i32(i32, i32) declare @llvm.get.active.lane.mask.nxv2i1.i32(i32, i32) +declare @llvm.get.active.lane.mask.nxv1i1.i32(i32, i32) declare @llvm.get.active.lane.mask.nxv64i1.i64(i64, i64) declare @llvm.get.active.lane.mask.nxv32i1.i64(i64, i64) From eecc4a61f6a59b96b6654be3bdfa705b23391ddb Mon Sep 17 00:00:00 2001 From: Kerry McLaughlin Date: Thu, 22 May 2025 13:36:59 +0000 Subject: [PATCH 5/5] - Renamed nxv1i1 test to lane_mask_nxv1i1_i32 --- llvm/test/CodeGen/AArch64/active_lane_mask.ll | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/test/CodeGen/AArch64/active_lane_mask.ll b/llvm/test/CodeGen/AArch64/active_lane_mask.ll index c5e7d1265ef25..fd34a1498bcd2 100644 --- a/llvm/test/CodeGen/AArch64/active_lane_mask.ll +++ b/llvm/test/CodeGen/AArch64/active_lane_mask.ll @@ -174,8 +174,8 @@ define @lane_mask_nxv7i1_i64(i64 %index, i64 %TC) { ret %active.lane.mask } -define @lane_mask_nxv1i1_i8(i32 %index, i32 %TC) { -; CHECK-LABEL: lane_mask_nxv1i1_i8: +define @lane_mask_nxv1i1_i32(i32 %index, i32 %TC) { +; CHECK-LABEL: lane_mask_nxv1i1_i32: ; CHECK: // %bb.0: ; CHECK-NEXT: index z0.s, #0, #1 ; CHECK-NEXT: mov z1.s, w0