diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h index 76d51ab819f44..4323f41182477 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -249,6 +249,18 @@ class CombinerHelper { /// or an implicit_def if \p Ops is empty. void applyCombineShuffleConcat(MachineInstr &MI, SmallVector &Ops); + /// Check if an instruction whose operations can be represented + /// by a mapping from one index to another in a vector can be replaced + /// by another operations. + /// \p Ops will contain the operands to produce the flattened + /// operation. + /// \p Mask is an array to numbers that represent the order that + /// the elements of \p SrcRegs will be put into \p DstReg. + bool analysePatternVectorMask(MachineInstr &MI, SmallVectorImpl &Ops, + const Register DstReg, + const std::pair SrcRegs, + ArrayRef Mask); + /// Try to combine G_SHUFFLE_VECTOR into G_CONCAT_VECTORS. /// Returns true if MI changed. /// @@ -580,6 +592,12 @@ class CombinerHelper { bool matchExtractVecEltBuildVec(MachineInstr &MI, Register &Reg); void applyExtractVecEltBuildVec(MachineInstr &MI, Register &Reg); + /// Combine extracts of two different arrays into one build vector into a + /// shuffle vector. + bool + matchCombineExtractToShuffle(MachineInstr &MI, SmallVectorImpl &Ops, + std::pair &VectorRegisters); + bool matchExtractAllEltsFromBuildVector( MachineInstr &MI, SmallVectorImpl> &MatchInfo); diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td index f838c6e62a2ce..e5ef655c4f1ca 100644 --- a/llvm/include/llvm/Target/GlobalISel/Combine.td +++ b/llvm/include/llvm/Target/GlobalISel/Combine.td @@ -905,6 +905,310 @@ def extract_vec_elt_build_vec : GICombineRule< [{ return Helper.matchExtractVecEltBuildVec(*${root}, ${matchinfo}); }]), (apply [{ Helper.applyExtractVecEltBuildVec(*${root}, ${matchinfo}); }])>; +def extract_vector_register_sources_matchinfo : + GIDefMatchData<"SmallVector">; +def vector_reg_pair_matchinfo : + GIDefMatchData<"std::pair">; +def extract_vector_element_build_vector_to_shuffle_vector2 : GICombineRule< + (defs root:$root, extract_vector_register_sources_matchinfo:$matchinfo, vector_reg_pair_matchinfo:$regpair), + (match (G_EXTRACT_VECTOR_ELT $el2, $vec2, $idx), + (G_EXTRACT_VECTOR_ELT $el1, $vec1, $idx), + (G_BUILD_VECTOR $root, $el1, $el2):$build, + [{ return Helper.matchCombineExtractToShuffle(*${build}, ${matchinfo}, ${regpair}); }]), + (apply [{ Helper.applyCombineShuffleVector(*${build}, ${matchinfo}); }])>; + +def extract_vector_element_build_vector_to_shuffle_vector4 : GICombineRule< + (defs root:$root, extract_vector_register_sources_matchinfo:$matchinfo, vector_reg_pair_matchinfo:$regpair), + (match (G_EXTRACT_VECTOR_ELT $el4, $vec4, $idx4), + (G_EXTRACT_VECTOR_ELT $el3, $vec3, $idx3), + (G_EXTRACT_VECTOR_ELT $el2, $vec2, $idx2), + (G_EXTRACT_VECTOR_ELT $el1, $vec1, $idx1), + (G_BUILD_VECTOR $root, $el1, $el2, $el3, $el4):$build, + [{ return Helper.matchCombineExtractToShuffle(*${build}, ${matchinfo}, ${regpair}); }]), + (apply [{ Helper.applyCombineShuffleVector(*${build}, ${matchinfo}); }])>; + +def extract_vector_element_build_vector_to_shuffle_vector8 : GICombineRule< + (defs root:$root, extract_vector_register_sources_matchinfo:$matchinfo, vector_reg_pair_matchinfo:$regpair), + (match + (G_EXTRACT_VECTOR_ELT $el8, $vec8, $idx8), + (G_EXTRACT_VECTOR_ELT $el7, $vec7, $idx7), + (G_EXTRACT_VECTOR_ELT $el6, $vec6, $idx6), + (G_EXTRACT_VECTOR_ELT $el5, $vec5, $idx5), + (G_EXTRACT_VECTOR_ELT $el4, $vec4, $idx4), + (G_EXTRACT_VECTOR_ELT $el3, $vec3, $idx3), + (G_EXTRACT_VECTOR_ELT $el2, $vec2, $idx2), + (G_EXTRACT_VECTOR_ELT $el1, $vec1, $idx1), + (G_BUILD_VECTOR $root, $el1, $el2, $el3, $el4, $el5, $el6, $el7, $el8):$build, + [{ return Helper.matchCombineExtractToShuffle(*${build}, ${matchinfo}, ${regpair}); }]), + (apply [{ Helper.applyCombineShuffleVector(*${build}, ${matchinfo}); }])>; + +def extract_vector_element_build_vector_to_shuffle_vector16 : GICombineRule< + (defs root:$root, extract_vector_register_sources_matchinfo:$matchinfo, vector_reg_pair_matchinfo:$regpair), + (match + (G_EXTRACT_VECTOR_ELT $el16, $vec16, $idx16), + (G_EXTRACT_VECTOR_ELT $el15, $vec15, $idx15), + (G_EXTRACT_VECTOR_ELT $el14, $vec14, $idx14), + (G_EXTRACT_VECTOR_ELT $el13, $vec13, $idx13), + (G_EXTRACT_VECTOR_ELT $el12, $vec12, $idx12), + (G_EXTRACT_VECTOR_ELT $el11, $vec11, $idx11), + (G_EXTRACT_VECTOR_ELT $el10, $vec10, $idx10), + (G_EXTRACT_VECTOR_ELT $el9, $vec9, $idx9), + (G_EXTRACT_VECTOR_ELT $el8, $vec8, $idx8), + (G_EXTRACT_VECTOR_ELT $el7, $vec7, $idx7), + (G_EXTRACT_VECTOR_ELT $el6, $vec6, $idx6), + (G_EXTRACT_VECTOR_ELT $el5, $vec5, $idx5), + (G_EXTRACT_VECTOR_ELT $el4, $vec4, $idx4), + (G_EXTRACT_VECTOR_ELT $el3, $vec3, $idx3), + (G_EXTRACT_VECTOR_ELT $el2, $vec2, $idx2), + (G_EXTRACT_VECTOR_ELT $el1, $vec1, $idx1), + (G_BUILD_VECTOR $root, $el1, $el2, $el3, $el4, $el5, $el6, $el7, $el8, $el9, $el10, $el11, $el12, $el13, $el14, $el15, $el16):$build, + [{ return Helper.matchCombineExtractToShuffle(*${build}, ${matchinfo}, ${regpair}); }]), + (apply [{ Helper.applyCombineShuffleVector(*${build}, ${matchinfo}); }])>; + +def extract_vector_element_build_vector_to_shuffle_vector32 : GICombineRule< + (defs root:$root, extract_vector_register_sources_matchinfo:$matchinfo, vector_reg_pair_matchinfo:$regpair), + (match + (G_EXTRACT_VECTOR_ELT $el32, $vec32, $idx32), + (G_EXTRACT_VECTOR_ELT $el31, $vec31, $idx31), + (G_EXTRACT_VECTOR_ELT $el30, $vec30, $idx30), + (G_EXTRACT_VECTOR_ELT $el29, $vec29, $idx29), + (G_EXTRACT_VECTOR_ELT $el28, $vec28, $idx28), + (G_EXTRACT_VECTOR_ELT $el27, $vec27, $idx27), + (G_EXTRACT_VECTOR_ELT $el26, $vec26, $idx26), + (G_EXTRACT_VECTOR_ELT $el25, $vec25, $idx25), + (G_EXTRACT_VECTOR_ELT $el24, $vec24, $idx24), + (G_EXTRACT_VECTOR_ELT $el23, $vec23, $idx23), + (G_EXTRACT_VECTOR_ELT $el22, $vec22, $idx22), + (G_EXTRACT_VECTOR_ELT $el21, $vec21, $idx21), + (G_EXTRACT_VECTOR_ELT $el20, $vec20, $idx20), + (G_EXTRACT_VECTOR_ELT $el19, $vec19, $idx19), + (G_EXTRACT_VECTOR_ELT $el18, $vec18, $idx18), + (G_EXTRACT_VECTOR_ELT $el17, $vec17, $idx17), + (G_EXTRACT_VECTOR_ELT $el16, $vec16, $idx16), + (G_EXTRACT_VECTOR_ELT $el15, $vec15, $idx15), + (G_EXTRACT_VECTOR_ELT $el14, $vec14, $idx14), + (G_EXTRACT_VECTOR_ELT $el13, $vec13, $idx13), + (G_EXTRACT_VECTOR_ELT $el12, $vec12, $idx12), + (G_EXTRACT_VECTOR_ELT $el11, $vec11, $idx11), + (G_EXTRACT_VECTOR_ELT $el10, $vec10, $idx10), + (G_EXTRACT_VECTOR_ELT $el9, $vec9, $idx9), + (G_EXTRACT_VECTOR_ELT $el8, $vec8, $idx8), + (G_EXTRACT_VECTOR_ELT $el7, $vec7, $idx7), + (G_EXTRACT_VECTOR_ELT $el6, $vec6, $idx6), + (G_EXTRACT_VECTOR_ELT $el5, $vec5, $idx5), + (G_EXTRACT_VECTOR_ELT $el4, $vec4, $idx4), + (G_EXTRACT_VECTOR_ELT $el3, $vec3, $idx3), + (G_EXTRACT_VECTOR_ELT $el2, $vec2, $idx2), + (G_EXTRACT_VECTOR_ELT $el1, $vec1, $idx1), + (G_BUILD_VECTOR $root, $el1, $el2, $el3, $el4, $el5, $el6, $el7, $el8, $el9, $el10, $el11, $el12, $el13, $el14, $el15, $el16, $el17, $el18, $el19, $el20, $el21, $el22, $el23, $el24, $el25, $el26, $el27, $el28, $el29, $el30, $el31, $el32):$build, + [{ return Helper.matchCombineExtractToShuffle(*${build}, ${matchinfo}, ${regpair}); }]), + (apply [{ Helper.applyCombineShuffleVector(*${build}, ${matchinfo}); }])>; + +def extract_vector_element_build_vector_to_shuffle_vector64 : GICombineRule< + (defs root:$root, extract_vector_register_sources_matchinfo:$matchinfo, vector_reg_pair_matchinfo:$regpair), + (match (G_EXTRACT_VECTOR_ELT $el64, $vec64, $idx64), + (G_EXTRACT_VECTOR_ELT $el63, $vec63, $idx63), + (G_EXTRACT_VECTOR_ELT $el62, $vec62, $idx62), + (G_EXTRACT_VECTOR_ELT $el61, $vec61, $idx61), + (G_EXTRACT_VECTOR_ELT $el60, $vec60, $idx60), + (G_EXTRACT_VECTOR_ELT $el59, $vec59, $idx59), + (G_EXTRACT_VECTOR_ELT $el58, $vec58, $idx58), + (G_EXTRACT_VECTOR_ELT $el57, $vec57, $idx57), + (G_EXTRACT_VECTOR_ELT $el56, $vec56, $idx56), + (G_EXTRACT_VECTOR_ELT $el55, $vec55, $idx55), + (G_EXTRACT_VECTOR_ELT $el54, $vec54, $idx54), + (G_EXTRACT_VECTOR_ELT $el53, $vec53, $idx53), + (G_EXTRACT_VECTOR_ELT $el52, $vec52, $idx52), + (G_EXTRACT_VECTOR_ELT $el51, $vec51, $idx51), + (G_EXTRACT_VECTOR_ELT $el50, $vec50, $idx50), + (G_EXTRACT_VECTOR_ELT $el49, $vec49, $idx49), + (G_EXTRACT_VECTOR_ELT $el48, $vec48, $idx48), + (G_EXTRACT_VECTOR_ELT $el47, $vec47, $idx47), + (G_EXTRACT_VECTOR_ELT $el46, $vec46, $idx46), + (G_EXTRACT_VECTOR_ELT $el45, $vec45, $idx45), + (G_EXTRACT_VECTOR_ELT $el44, $vec44, $idx44), + (G_EXTRACT_VECTOR_ELT $el43, $vec43, $idx43), + (G_EXTRACT_VECTOR_ELT $el42, $vec42, $idx42), + (G_EXTRACT_VECTOR_ELT $el41, $vec41, $idx41), + (G_EXTRACT_VECTOR_ELT $el40, $vec40, $idx40), + (G_EXTRACT_VECTOR_ELT $el39, $vec39, $idx39), + (G_EXTRACT_VECTOR_ELT $el38, $vec38, $idx38), + (G_EXTRACT_VECTOR_ELT $el37, $vec37, $idx37), + (G_EXTRACT_VECTOR_ELT $el36, $vec36, $idx36), + (G_EXTRACT_VECTOR_ELT $el35, $vec35, $idx35), + (G_EXTRACT_VECTOR_ELT $el34, $vec34, $idx34), + (G_EXTRACT_VECTOR_ELT $el33, $vec33, $idx33), + (G_EXTRACT_VECTOR_ELT $el32, $vec32, $idx32), + (G_EXTRACT_VECTOR_ELT $el31, $vec31, $idx31), + (G_EXTRACT_VECTOR_ELT $el30, $vec30, $idx30), + (G_EXTRACT_VECTOR_ELT $el29, $vec29, $idx29), + (G_EXTRACT_VECTOR_ELT $el28, $vec28, $idx28), + (G_EXTRACT_VECTOR_ELT $el27, $vec27, $idx27), + (G_EXTRACT_VECTOR_ELT $el26, $vec26, $idx26), + (G_EXTRACT_VECTOR_ELT $el25, $vec25, $idx25), + (G_EXTRACT_VECTOR_ELT $el24, $vec24, $idx24), + (G_EXTRACT_VECTOR_ELT $el23, $vec23, $idx23), + (G_EXTRACT_VECTOR_ELT $el22, $vec22, $idx22), + (G_EXTRACT_VECTOR_ELT $el21, $vec21, $idx21), + (G_EXTRACT_VECTOR_ELT $el20, $vec20, $idx20), + (G_EXTRACT_VECTOR_ELT $el19, $vec19, $idx19), + (G_EXTRACT_VECTOR_ELT $el18, $vec18, $idx18), + (G_EXTRACT_VECTOR_ELT $el17, $vec17, $idx17), + (G_EXTRACT_VECTOR_ELT $el16, $vec16, $idx16), + (G_EXTRACT_VECTOR_ELT $el15, $vec15, $idx15), + (G_EXTRACT_VECTOR_ELT $el14, $vec14, $idx14), + (G_EXTRACT_VECTOR_ELT $el13, $vec13, $idx13), + (G_EXTRACT_VECTOR_ELT $el12, $vec12, $idx12), + (G_EXTRACT_VECTOR_ELT $el11, $vec11, $idx11), + (G_EXTRACT_VECTOR_ELT $el10, $vec10, $idx10), + (G_EXTRACT_VECTOR_ELT $el9, $vec9, $idx9), + (G_EXTRACT_VECTOR_ELT $el8, $vec8, $idx8), + (G_EXTRACT_VECTOR_ELT $el7, $vec7, $idx7), + (G_EXTRACT_VECTOR_ELT $el6, $vec6, $idx6), + (G_EXTRACT_VECTOR_ELT $el5, $vec5, $idx5), + (G_EXTRACT_VECTOR_ELT $el4, $vec4, $idx4), + (G_EXTRACT_VECTOR_ELT $el3, $vec3, $idx3), + (G_EXTRACT_VECTOR_ELT $el2, $vec2, $idx2), + (G_EXTRACT_VECTOR_ELT $el1, $vec1, $idx1), + (G_BUILD_VECTOR $root, $el1, $el2, $el3, $el4, $el5, $el6, $el7, $el8, $el9, $el10, $el11, $el12, $el13, $el14, $el15, $el16, $el17, $el18, $el19, $el20, $el21, $el22, $el23, $el24, $el25, $el26, $el27, $el28, $el29, $el30, $el31, $el32, $el33, $el34, $el35, $el36, $el37, $el38, $el39, $el40, $el41, $el42, $el43, $el44, $el45, $el46, $el47, $el48, $el49, $el50, $el51, $el52, $el53, $el54, $el55, $el56, $el57, $el58, $el59, $el60, $el61, $el62, $el63, $el64):$build, + [{ return Helper.matchCombineExtractToShuffle(*${build}, ${matchinfo}, ${regpair}); }]), + (apply [{ Helper.applyCombineShuffleVector(*${build}, ${matchinfo}); }])>; + + +def extract_vector_element_build_vector_to_shuffle_vector128 : GICombineRule< + (defs root:$root, extract_vector_register_sources_matchinfo:$matchinfo, vector_reg_pair_matchinfo:$regpair), + (match (G_EXTRACT_VECTOR_ELT $el128, $vec128, $idx128), + (G_EXTRACT_VECTOR_ELT $el127, $vec127, $idx127), + (G_EXTRACT_VECTOR_ELT $el126, $vec126, $idx126), + (G_EXTRACT_VECTOR_ELT $el125, $vec125, $idx125), + (G_EXTRACT_VECTOR_ELT $el124, $vec124, $idx124), + (G_EXTRACT_VECTOR_ELT $el123, $vec123, $idx123), + (G_EXTRACT_VECTOR_ELT $el122, $vec122, $idx122), + (G_EXTRACT_VECTOR_ELT $el121, $vec121, $idx121), + (G_EXTRACT_VECTOR_ELT $el120, $vec120, $idx120), + (G_EXTRACT_VECTOR_ELT $el119, $vec119, $idx119), + (G_EXTRACT_VECTOR_ELT $el118, $vec118, $idx118), + (G_EXTRACT_VECTOR_ELT $el117, $vec117, $idx117), + (G_EXTRACT_VECTOR_ELT $el116, $vec116, $idx116), + (G_EXTRACT_VECTOR_ELT $el115, $vec115, $idx115), + (G_EXTRACT_VECTOR_ELT $el114, $vec114, $idx114), + (G_EXTRACT_VECTOR_ELT $el113, $vec113, $idx113), + (G_EXTRACT_VECTOR_ELT $el112, $vec112, $idx112), + (G_EXTRACT_VECTOR_ELT $el111, $vec111, $idx111), + (G_EXTRACT_VECTOR_ELT $el110, $vec110, $idx110), + (G_EXTRACT_VECTOR_ELT $el109, $vec109, $idx109), + (G_EXTRACT_VECTOR_ELT $el108, $vec108, $idx108), + (G_EXTRACT_VECTOR_ELT $el107, $vec107, $idx107), + (G_EXTRACT_VECTOR_ELT $el106, $vec106, $idx106), + (G_EXTRACT_VECTOR_ELT $el105, $vec105, $idx105), + (G_EXTRACT_VECTOR_ELT $el104, $vec104, $idx104), + (G_EXTRACT_VECTOR_ELT $el103, $vec103, $idx103), + (G_EXTRACT_VECTOR_ELT $el102, $vec102, $idx102), + (G_EXTRACT_VECTOR_ELT $el101, $vec101, $idx101), + (G_EXTRACT_VECTOR_ELT $el100, $vec100, $idx100), + (G_EXTRACT_VECTOR_ELT $el99, $vec99, $idx99), + (G_EXTRACT_VECTOR_ELT $el98, $vec98, $idx98), + (G_EXTRACT_VECTOR_ELT $el97, $vec97, $idx97), + (G_EXTRACT_VECTOR_ELT $el96, $vec96, $idx96), + (G_EXTRACT_VECTOR_ELT $el95, $vec95, $idx95), + (G_EXTRACT_VECTOR_ELT $el94, $vec94, $idx94), + (G_EXTRACT_VECTOR_ELT $el93, $vec93, $idx93), + (G_EXTRACT_VECTOR_ELT $el92, $vec92, $idx92), + (G_EXTRACT_VECTOR_ELT $el91, $vec91, $idx91), + (G_EXTRACT_VECTOR_ELT $el90, $vec90, $idx90), + (G_EXTRACT_VECTOR_ELT $el89, $vec89, $idx89), + (G_EXTRACT_VECTOR_ELT $el88, $vec88, $idx88), + (G_EXTRACT_VECTOR_ELT $el87, $vec87, $idx87), + (G_EXTRACT_VECTOR_ELT $el86, $vec86, $idx86), + (G_EXTRACT_VECTOR_ELT $el85, $vec85, $idx85), + (G_EXTRACT_VECTOR_ELT $el84, $vec84, $idx84), + (G_EXTRACT_VECTOR_ELT $el83, $vec83, $idx83), + (G_EXTRACT_VECTOR_ELT $el82, $vec82, $idx82), + (G_EXTRACT_VECTOR_ELT $el81, $vec81, $idx81), + (G_EXTRACT_VECTOR_ELT $el80, $vec80, $idx80), + (G_EXTRACT_VECTOR_ELT $el79, $vec79, $idx79), + (G_EXTRACT_VECTOR_ELT $el78, $vec78, $idx78), + (G_EXTRACT_VECTOR_ELT $el77, $vec77, $idx77), + (G_EXTRACT_VECTOR_ELT $el76, $vec76, $idx76), + (G_EXTRACT_VECTOR_ELT $el75, $vec75, $idx75), + (G_EXTRACT_VECTOR_ELT $el74, $vec74, $idx74), + (G_EXTRACT_VECTOR_ELT $el73, $vec73, $idx73), + (G_EXTRACT_VECTOR_ELT $el72, $vec72, $idx72), + (G_EXTRACT_VECTOR_ELT $el71, $vec71, $idx71), + (G_EXTRACT_VECTOR_ELT $el70, $vec70, $idx70), + (G_EXTRACT_VECTOR_ELT $el69, $vec69, $idx69), + (G_EXTRACT_VECTOR_ELT $el68, $vec68, $idx68), + (G_EXTRACT_VECTOR_ELT $el67, $vec67, $idx67), + (G_EXTRACT_VECTOR_ELT $el66, $vec66, $idx66), + (G_EXTRACT_VECTOR_ELT $el65, $vec65, $idx65), + (G_EXTRACT_VECTOR_ELT $el64, $vec64, $idx64), + (G_EXTRACT_VECTOR_ELT $el63, $vec63, $idx63), + (G_EXTRACT_VECTOR_ELT $el62, $vec62, $idx62), + (G_EXTRACT_VECTOR_ELT $el61, $vec61, $idx61), + (G_EXTRACT_VECTOR_ELT $el60, $vec60, $idx60), + (G_EXTRACT_VECTOR_ELT $el59, $vec59, $idx59), + (G_EXTRACT_VECTOR_ELT $el58, $vec58, $idx58), + (G_EXTRACT_VECTOR_ELT $el57, $vec57, $idx57), + (G_EXTRACT_VECTOR_ELT $el56, $vec56, $idx56), + (G_EXTRACT_VECTOR_ELT $el55, $vec55, $idx55), + (G_EXTRACT_VECTOR_ELT $el54, $vec54, $idx54), + (G_EXTRACT_VECTOR_ELT $el53, $vec53, $idx53), + (G_EXTRACT_VECTOR_ELT $el52, $vec52, $idx52), + (G_EXTRACT_VECTOR_ELT $el51, $vec51, $idx51), + (G_EXTRACT_VECTOR_ELT $el50, $vec50, $idx50), + (G_EXTRACT_VECTOR_ELT $el49, $vec49, $idx49), + (G_EXTRACT_VECTOR_ELT $el48, $vec48, $idx48), + (G_EXTRACT_VECTOR_ELT $el47, $vec47, $idx47), + (G_EXTRACT_VECTOR_ELT $el46, $vec46, $idx46), + (G_EXTRACT_VECTOR_ELT $el45, $vec45, $idx45), + (G_EXTRACT_VECTOR_ELT $el44, $vec44, $idx44), + (G_EXTRACT_VECTOR_ELT $el43, $vec43, $idx43), + (G_EXTRACT_VECTOR_ELT $el42, $vec42, $idx42), + (G_EXTRACT_VECTOR_ELT $el41, $vec41, $idx41), + (G_EXTRACT_VECTOR_ELT $el40, $vec40, $idx40), + (G_EXTRACT_VECTOR_ELT $el39, $vec39, $idx39), + (G_EXTRACT_VECTOR_ELT $el38, $vec38, $idx38), + (G_EXTRACT_VECTOR_ELT $el37, $vec37, $idx37), + (G_EXTRACT_VECTOR_ELT $el36, $vec36, $idx36), + (G_EXTRACT_VECTOR_ELT $el35, $vec35, $idx35), + (G_EXTRACT_VECTOR_ELT $el34, $vec34, $idx34), + (G_EXTRACT_VECTOR_ELT $el33, $vec33, $idx33), + (G_EXTRACT_VECTOR_ELT $el32, $vec32, $idx32), + (G_EXTRACT_VECTOR_ELT $el31, $vec31, $idx31), + (G_EXTRACT_VECTOR_ELT $el30, $vec30, $idx30), + (G_EXTRACT_VECTOR_ELT $el29, $vec29, $idx29), + (G_EXTRACT_VECTOR_ELT $el28, $vec28, $idx28), + (G_EXTRACT_VECTOR_ELT $el27, $vec27, $idx27), + (G_EXTRACT_VECTOR_ELT $el26, $vec26, $idx26), + (G_EXTRACT_VECTOR_ELT $el25, $vec25, $idx25), + (G_EXTRACT_VECTOR_ELT $el24, $vec24, $idx24), + (G_EXTRACT_VECTOR_ELT $el23, $vec23, $idx23), + (G_EXTRACT_VECTOR_ELT $el22, $vec22, $idx22), + (G_EXTRACT_VECTOR_ELT $el21, $vec21, $idx21), + (G_EXTRACT_VECTOR_ELT $el20, $vec20, $idx20), + (G_EXTRACT_VECTOR_ELT $el19, $vec19, $idx19), + (G_EXTRACT_VECTOR_ELT $el18, $vec18, $idx18), + (G_EXTRACT_VECTOR_ELT $el17, $vec17, $idx17), + (G_EXTRACT_VECTOR_ELT $el16, $vec16, $idx16), + (G_EXTRACT_VECTOR_ELT $el15, $vec15, $idx15), + (G_EXTRACT_VECTOR_ELT $el14, $vec14, $idx14), + (G_EXTRACT_VECTOR_ELT $el13, $vec13, $idx13), + (G_EXTRACT_VECTOR_ELT $el12, $vec12, $idx12), + (G_EXTRACT_VECTOR_ELT $el11, $vec11, $idx11), + (G_EXTRACT_VECTOR_ELT $el10, $vec10, $idx10), + (G_EXTRACT_VECTOR_ELT $el9, $vec9, $idx9), + (G_EXTRACT_VECTOR_ELT $el8, $vec8, $idx8), + (G_EXTRACT_VECTOR_ELT $el7, $vec7, $idx7), + (G_EXTRACT_VECTOR_ELT $el6, $vec6, $idx6), + (G_EXTRACT_VECTOR_ELT $el5, $vec5, $idx5), + (G_EXTRACT_VECTOR_ELT $el4, $vec4, $idx4), + (G_EXTRACT_VECTOR_ELT $el3, $vec3, $idx3), + (G_EXTRACT_VECTOR_ELT $el2, $vec2, $idx2), + (G_EXTRACT_VECTOR_ELT $el1, $vec1, $idx1), + (G_BUILD_VECTOR $root, $el1, $el2, $el3, $el4, $el5, $el6, $el7, $el8, $el9, $el10, $el11, $el12, $el13, $el14, $el15, $el16, $el17, $el18, $el19, $el20, $el21, $el22, $el23, $el24, $el25, $el26, $el27, $el28, $el29, $el30, $el31, $el32, $el33, $el34, $el35, $el36, $el37, $el38, $el39, $el40, $el41, $el42, $el43, $el44, $el45, $el46, $el47, $el48, $el49, $el50, $el51, $el52, $el53, $el54, $el55, $el56, $el57, $el58, $el59, $el60, $el61, $el62, $el63, $el64, $el65, $el66, $el67, $el68, $el69, $el70, $el71, $el72, $el73, $el74, $el75, $el76, $el77, $el78, $el79, $el80, $el81, $el82, $el83, $el84, $el85, $el86, $el87, $el88, $el89, $el90, $el91, $el92, $el93, $el94, $el95, $el96, $el97, $el98, $el99, $el100, $el101, $el102, $el103, $el104, $el105, $el106, $el107, $el108, $el109, $el110, $el111, $el112, $el113, $el114, $el115, $el116, $el117, $el118, $el119, $el120, $el121, $el122, $el123, $el124, $el125, $el126, $el127, $el128):$build, + [{ return Helper.matchCombineExtractToShuffle(*${build}, ${matchinfo}, ${regpair}); }]), + (apply [{ Helper.applyCombineShuffleVector(*${build}, ${matchinfo}); }])>; + // Fold away full elt extracts from a build_vector. def extract_all_elts_from_build_vector_matchinfo : GIDefMatchData<"SmallVector>">; @@ -916,6 +1220,13 @@ def extract_all_elts_from_build_vector : GICombineRule< def extract_vec_elt_combines : GICombineGroup<[ extract_vec_elt_build_vec, + extract_vector_element_build_vector_to_shuffle_vector2, + extract_vector_element_build_vector_to_shuffle_vector4, + extract_vector_element_build_vector_to_shuffle_vector8, + extract_vector_element_build_vector_to_shuffle_vector16, + extract_vector_element_build_vector_to_shuffle_vector32, + extract_vector_element_build_vector_to_shuffle_vector64, + extract_vector_element_build_vector_to_shuffle_vector128, extract_all_elts_from_build_vector]>; def funnel_shift_from_or_shift : GICombineRule< @@ -1916,7 +2227,7 @@ def all_combines : GICombineGroup<[integer_reassoc_combines, trivial_combines, constant_fold_cast_op, fabs_fneg_fold, intdiv_combines, mulh_combines, redundant_neg_operands, and_or_disjoint_mask, fma_combines, fold_binop_into_select, - sub_add_reg, select_to_minmax, + sub_add_reg, select_to_minmax, fsub_to_fneg, commute_constant_to_rhs, match_ands, match_ors, combine_concat_vector, match_addos, sext_trunc, zext_trunc, prefer_sign_combines, combine_shuffle_concat, diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp index c279289f9161b..940106f02ce9e 100644 --- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp @@ -38,8 +38,10 @@ #include "llvm/Support/MathExtras.h" #include "llvm/Target/TargetMachine.h" #include +#include #include #include +#include #define DEBUG_TYPE "gi-combiner" @@ -472,39 +474,21 @@ bool CombinerHelper::tryCombineShuffleVector(MachineInstr &MI) { return false; } -bool CombinerHelper::matchCombineShuffleVector(MachineInstr &MI, - SmallVectorImpl &Ops) { - assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR && - "Invalid instruction kind"); - LLT DstType = MRI.getType(MI.getOperand(0).getReg()); - Register Src1 = MI.getOperand(1).getReg(); - LLT SrcType = MRI.getType(Src1); - // As bizarre as it may look, shuffle vector can actually produce - // scalar! This is because at the IR level a <1 x ty> shuffle - // vector is perfectly valid. - unsigned DstNumElts = DstType.isVector() ? DstType.getNumElements() : 1; - unsigned SrcNumElts = SrcType.isVector() ? SrcType.getNumElements() : 1; - - // If the resulting vector is smaller than the size of the source - // vectors being concatenated, we won't be able to replace the - // shuffle vector into a concat_vectors. - // - // Note: We may still be able to produce a concat_vectors fed by - // extract_vector_elt and so on. It is less clear that would - // be better though, so don't bother for now. - // - // If the destination is a scalar, the size of the sources doesn't - // matter. we will lower the shuffle to a plain copy. This will - // work only if the source and destination have the same size. But - // that's covered by the next condition. - // - // TODO: If the size between the source and destination don't match - // we could still emit an extract vector element in that case. - if (DstNumElts < 2 * SrcNumElts && DstNumElts != 1) - return false; +/// Helper function for instruction sequences that can be represented by a +/// mapping from one index to another. E.x. shufflevectors and insert/extract +/// +/// Checks whether the input \p Mask refers to a pre-known sequence and is a +/// valid target to be replaced. Currently supports only concatenation. +bool CombinerHelper::analysePatternVectorMask( + MachineInstr &MI, SmallVectorImpl &Ops, const Register DstReg, + const std::pair SrcRegs, ArrayRef Mask) { + const LLT DstTy = MRI.getType(DstReg); + const LLT SrcTy = MRI.getType(SrcRegs.first); // Check that the shuffle mask can be broken evenly between the // different sources. + const unsigned DstNumElts = DstTy.isVector() ? DstTy.getNumElements() : 1; + const unsigned SrcNumElts = SrcTy.isVector() ? SrcTy.getNumElements() : 1; if (DstNumElts % SrcNumElts != 0) return false; @@ -513,7 +497,6 @@ bool CombinerHelper::matchCombineShuffleVector(MachineInstr &MI, // vectors. unsigned NumConcat = DstNumElts / SrcNumElts; SmallVector ConcatSrcs(NumConcat, -1); - ArrayRef Mask = MI.getOperand(3).getShuffleMask(); for (unsigned i = 0; i != DstNumElts; ++i) { int Idx = Mask[i]; // Undef value. @@ -532,22 +515,58 @@ bool CombinerHelper::matchCombineShuffleVector(MachineInstr &MI, // The shuffle is concatenating multiple vectors together. // Collect the different operands for that. Register UndefReg; - Register Src2 = MI.getOperand(2).getReg(); for (auto Src : ConcatSrcs) { if (Src < 0) { if (!UndefReg) { Builder.setInsertPt(*MI.getParent(), MI); - UndefReg = Builder.buildUndef(SrcType).getReg(0); + UndefReg = Builder.buildUndef(SrcTy).getReg(0); } Ops.push_back(UndefReg); } else if (Src == 0) - Ops.push_back(Src1); + Ops.push_back(SrcRegs.first); else - Ops.push_back(Src2); + Ops.push_back(SrcRegs.second); } return true; } +bool CombinerHelper::matchCombineShuffleVector(MachineInstr &MI, + SmallVectorImpl &Ops) { + assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR && + "Invalid instruction kind"); + LLT DstType = MRI.getType(MI.getOperand(0).getReg()); + Register Src1 = MI.getOperand(1).getReg(); + LLT SrcType = MRI.getType(Src1); + // As bizarre as it may look, shuffle vector can actually produce + // scalar! This is because at the IR level a <1 x ty> shuffle + // vector is perfectly valid. + unsigned DstNumElts = DstType.isVector() ? DstType.getNumElements() : 1; + unsigned SrcNumElts = SrcType.isVector() ? SrcType.getNumElements() : 1; + + // If the resulting vector is smaller than the size of the source + // vectors being concatenated, we won't be able to replace the + // shuffle vector into a concat_vectors. + // + // Note: We may still be able to produce a concat_vectors fed by + // extract_vector_elt and so on. It is less clear that would + // be better though, so don't bother for now. + // + // If the destination is a scalar, the size of the sources doesn't + // matter. we will lower the shuffle to a plain copy. This will + // work only if the source and destination have the same size. But + // that's covered by the next condition. + // + // TODO: If the size between the source and destination don't match + // we could still emit an extract vector element in that case. + if (DstNumElts < 2 * SrcNumElts && DstNumElts != 1) + return false; + + return analysePatternVectorMask( + MI, Ops, MI.getOperand(0).getReg(), + std::make_pair(MI.getOperand(1).getReg(), MI.getOperand(2).getReg()), + MI.getOperand(3).getShuffleMask()); +} + void CombinerHelper::applyCombineShuffleVector(MachineInstr &MI, const ArrayRef Ops) { Register DstReg = MI.getOperand(0).getReg(); @@ -4205,6 +4224,73 @@ void CombinerHelper::applyExtractVecEltBuildVec(MachineInstr &MI, replaceSingleDefInstWithReg(MI, Reg); } +bool CombinerHelper::matchCombineExtractToShuffle( + MachineInstr &MI, SmallVectorImpl &Ops, + std::pair &VectorRegisters) { + const GBuildVector *Build = cast(&MI); + const unsigned SrcNumElts = + MRI.getType(MI.getOperand(0).getReg()).getNumElements(); + + // This combine tries to find all the build vectors whose source elements + // all originate from a G_EXTRACT_VECTOR_ELT from one or two donor vectors. + // E.g. + // %donor1(<2 x s32>) = COPY $d0 + // %donor2(<2 x s32>) = COPY $d1 + // %ext1 = G_EXTRACT_VECTOR_ELT %donor1, 0 + // %ext2 = G_EXTRACT_VECTOR_ELT %donor1, 1 + // %ext3 = G_EXTRACT_VECTOR_ELT %donor2, 0 + // %ext4 = G_EXTRACT_VECTOR_ELT %donor2, 1 + /// %vector = G_BUILD_VECTOR %ext1, %ext2, %ext3, %ext4 + // ==> + // replace with: + // %vector = G_SHUFFLE_VECTOR %donor1, %donor2, shufflemask(0, 1, 2, 3) + SmallSetVector RegisterVector; + SmallVector VectorMask; + const unsigned NumElements = Build->getNumSources(); + for (unsigned Index = 0; Index < NumElements; Index++) { + Register SrcReg = peekThroughBitcast(Build->getSourceReg(Index), MRI); + auto *ExtractInstr = getOpcodeDef(SrcReg, MRI); + if (!ExtractInstr) + return false; + + RegisterVector.insert(ExtractInstr->getVectorReg()); + + // For shufflemasks we need to know exactly what index to place each element + // so if it this build vector doesn't use exclusively constants than we + // can't replace with a shufflevector + auto Cst = getIConstantVRegVal(ExtractInstr->getIndexReg(), MRI); + if (!Cst) + return false; + + unsigned Idx = Cst->getZExtValue(); + if (ExtractInstr->getVectorReg() != RegisterVector.front()) { + Idx += SrcNumElts; + } + + VectorMask.emplace_back(Idx); + } + + // Create a pair so that we don't need to look for them later. This code is + // incorrect if we have more than two vectors in the set. Since we can only + // put two vectors in a shuffle, we reject any solution with more than two + // anyways. + VectorRegisters = + std::make_pair(RegisterVector.front(), RegisterVector.back()); + + // We check that they're the same type before running. We can also grow the + // smaller one tro the target size, but there isn't an elegant way to do that + // until we have a good lowerng for G_EXTRACT_SUBVECTOR. + // Apparently if they are the same, they don't necessary have the same type? + if (MRI.getType(VectorRegisters.first) != MRI.getType(VectorRegisters.second)) + return false; + + if (RegisterVector.size() > 2) + return false; + + return analysePatternVectorMask(MI, Ops, MI.getOperand(0).getReg(), + VectorRegisters, VectorMask); +} + bool CombinerHelper::matchExtractAllEltsFromBuildVector( MachineInstr &MI, SmallVectorImpl> &SrcDstPairs) { diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp index 59f2fc633f5de..1ddecefa17383 100644 --- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp +++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp @@ -772,8 +772,6 @@ MachineInstrBuilder MachineIRBuilder::buildShuffleVector(const DstOp &Res, LLT DstTy = Res.getLLTTy(*getMRI()); LLT Src1Ty = Src1.getLLTTy(*getMRI()); LLT Src2Ty = Src2.getLLTTy(*getMRI()); - assert((size_t)(Src1Ty.getNumElements() + Src2Ty.getNumElements()) >= - Mask.size()); assert(DstTy.getElementType() == Src1Ty.getElementType() && DstTy.getElementType() == Src2Ty.getElementType()); (void)DstTy; diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-build-vector.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-build-vector.mir index 93f6051c3bd3b..a231bc8f29ef6 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-build-vector.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-build-vector.mir @@ -151,3 +151,326 @@ body: | RET_ReallyLR implicit $x0 ... +--- +name: concat_buildvector_shuffle +tracksRegLiveness: true +body: | + bb.1: + liveins: $q0, $q1 + ; CHECK-LABEL: name: concat_buildvector_shuffle + ; CHECK: liveins: $q0, $q1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1 + ; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s32>) = G_CONCAT_VECTORS [[COPY]](<4 x s32>), [[COPY1]](<4 x s32>) + ; CHECK-NEXT: RET_ReallyLR implicit [[CONCAT_VECTORS]](<8 x s32>) + %0:_(<4 x s32>) = COPY $q0 + %1:_(<4 x s32>) = COPY $q1 + %2:_(s64) = G_CONSTANT i64 0 + %3:_(s64) = G_CONSTANT i64 1 + %4:_(s64) = G_CONSTANT i64 2 + %5:_(s64) = G_CONSTANT i64 3 + %10:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %2:_(s64) + %11:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %3:_(s64) + %12:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %4:_(s64) + %13:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %5:_(s64) + %14:_(s32) = G_EXTRACT_VECTOR_ELT %1:_(<4 x s32>), %2:_(s64) + %15:_(s32) = G_EXTRACT_VECTOR_ELT %1:_(<4 x s32>), %3:_(s64) + %16:_(s32) = G_EXTRACT_VECTOR_ELT %1:_(<4 x s32>), %4:_(s64) + %17:_(s32) = G_EXTRACT_VECTOR_ELT %1:_(<4 x s32>), %5:_(s64) + %18:_(<8 x s32>) = G_BUILD_VECTOR %10:_(s32), %11:_, %12:_, %13:_, %14:_, %15:_, %16:_, %17:_ + RET_ReallyLR implicit %18 +... +--- +name: interweave_buildvector_shuffle +tracksRegLiveness: true +body: | + bb.1: + liveins: $q0, $q1 + ; CHECK-LABEL: name: interweave_buildvector_shuffle + ; CHECK: liveins: $q0, $q1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1 + ; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s32>) = G_CONCAT_VECTORS [[COPY]](<4 x s32>), [[COPY1]](<4 x s32>) + ; CHECK-NEXT: RET_ReallyLR implicit [[CONCAT_VECTORS]](<8 x s32>) + %0:_(<4 x s32>) = COPY $q0 + %1:_(<4 x s32>) = COPY $q1 + %2:_(s64) = G_CONSTANT i64 0 + %3:_(s64) = G_CONSTANT i64 1 + %4:_(s64) = G_CONSTANT i64 2 + %5:_(s64) = G_CONSTANT i64 3 + %10:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %2:_(s64) + %11:_(s32) = G_EXTRACT_VECTOR_ELT %1:_(<4 x s32>), %3:_(s64) + %12:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %4:_(s64) + %13:_(s32) = G_EXTRACT_VECTOR_ELT %1:_(<4 x s32>), %5:_(s64) + %14:_(s32) = G_EXTRACT_VECTOR_ELT %1:_(<4 x s32>), %2:_(s64) + %15:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %3:_(s64) + %16:_(s32) = G_EXTRACT_VECTOR_ELT %1:_(<4 x s32>), %4:_(s64) + %17:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %5:_(s64) + %18:_(<8 x s32>) = G_BUILD_VECTOR %10:_(s32), %15:_, %12:_, %17:_, %14:_, %11:_, %16:_, %13:_ + RET_ReallyLR implicit %18 +... + +--- +name: interweave_same_size_as_dest_buildvector_shuffle +tracksRegLiveness: true +body: | + bb.1: + liveins: $q0 + ; CHECK-LABEL: name: interweave_same_size_as_dest_buildvector_shuffle + ; CHECK: liveins: $q0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0 + ; CHECK-NEXT: RET_ReallyLR implicit [[COPY]](<4 x s32>) + %0:_(<4 x s32>) = COPY $q0 + %1:_(s64) = G_CONSTANT i64 0 + %2:_(s64) = G_CONSTANT i64 1 + %3:_(s64) = G_CONSTANT i64 2 + %4:_(s64) = G_CONSTANT i64 3 + %10:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %1:_(s64) + %11:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %2:_(s64) + %12:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %3:_(s64) + %13:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %4:_(s64) + %14:_(<4 x s32>) = G_BUILD_VECTOR %10:_(s32), %11:_, %12:_, %13:_ + RET_ReallyLR implicit %14 +... +--- +name: half_size_as_dest_buildvector_shuffle +tracksRegLiveness: true +body: | + bb.1: + liveins: $q0 + ; CHECK-LABEL: name: half_size_as_dest_buildvector_shuffle + ; CHECK: liveins: $q0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0 + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1 + ; CHECK-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](<4 x s32>), [[C]](s64) + ; CHECK-NEXT: [[EVEC1:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](<4 x s32>), [[C1]](s64) + ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[EVEC]](s32), [[EVEC1]](s32) + ; CHECK-NEXT: RET_ReallyLR implicit [[BUILD_VECTOR]](<2 x s32>) + %0:_(<4 x s32>) = COPY $q0 + %1:_(s64) = G_CONSTANT i64 0 + %2:_(s64) = G_CONSTANT i64 1 + %10:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %1:_(s64) + %11:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %2:_(s64) + %12:_(<2 x s32>) = G_BUILD_VECTOR %10:_(s32), %11:_ + RET_ReallyLR implicit %12 +... +--- +name: concat_buildvector_shuffle_three_sources +tracksRegLiveness: true +body: | + bb.1: + liveins: $d0, $d1, $d2, $d3 + ; CHECK-LABEL: name: concat_buildvector_shuffle_three_sources + ; CHECK: liveins: $d0, $d1, $d2, $d3 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1 + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(<2 x s32>) = COPY $d2 + ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(<2 x s32>) = COPY $d3 + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1 + ; CHECK-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](<2 x s32>), [[C]](s64) + ; CHECK-NEXT: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF + ; CHECK-NEXT: [[EVEC1:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY1]](<2 x s32>), [[C]](s64) + ; CHECK-NEXT: [[EVEC2:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY2]](<2 x s32>), [[C]](s64) + ; CHECK-NEXT: [[EVEC3:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY3]](<2 x s32>), [[C]](s64) + ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[EVEC]](s32), [[DEF]](s32), [[EVEC1]](s32), [[DEF]](s32), [[EVEC2]](s32), [[DEF]](s32), [[EVEC3]](s32), [[DEF]](s32) + ; CHECK-NEXT: RET_ReallyLR implicit [[BUILD_VECTOR]](<8 x s32>) + %0:_(<2 x s32>) = COPY $d0 + %1:_(<2 x s32>) = COPY $d1 + %2:_(<2 x s32>) = COPY $d2 + %3:_(<2 x s32>) = COPY $d3 + %4:_(s64) = G_CONSTANT i64 1 + %5:_(s64) = G_CONSTANT i64 2 + %11:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<2 x s32>), %4:_(s64) + %12:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<2 x s32>), %5:_(s64) + %13:_(s32) = G_EXTRACT_VECTOR_ELT %1:_(<2 x s32>), %4:_(s64) + %14:_(s32) = G_EXTRACT_VECTOR_ELT %1:_(<2 x s32>), %5:_(s64) + %15:_(s32) = G_EXTRACT_VECTOR_ELT %2:_(<2 x s32>), %4:_(s64) + %16:_(s32) = G_EXTRACT_VECTOR_ELT %2:_(<2 x s32>), %5:_(s64) + %17:_(s32) = G_EXTRACT_VECTOR_ELT %3:_(<2 x s32>), %4:_(s64) + %18:_(s32) = G_EXTRACT_VECTOR_ELT %3:_(<2 x s32>), %5:_(s64) + %19:_(<8 x s32>) = G_BUILD_VECTOR %11:_(s32), %12:_, %13:_, %14:_, %15:_, %16:_, %17:_, %18:_ + RET_ReallyLR implicit %19 +... +--- +name: concat_buildvector_shuffle_different_element_size +tracksRegLiveness: true +body: | + bb.1: + liveins: $q0, $d0 + ; CHECK-LABEL: name: concat_buildvector_shuffle_different_element_size + ; CHECK: liveins: $q0, $d0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d0 + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1 + ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2 + ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 3 + ; CHECK-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](<4 x s32>), [[C]](s64) + ; CHECK-NEXT: [[EVEC1:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](<4 x s32>), [[C1]](s64) + ; CHECK-NEXT: [[EVEC2:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](<4 x s32>), [[C2]](s64) + ; CHECK-NEXT: [[EVEC3:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](<4 x s32>), [[C3]](s64) + ; CHECK-NEXT: [[EVEC4:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY1]](<2 x s32>), [[C]](s64) + ; CHECK-NEXT: [[EVEC5:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY1]](<2 x s32>), [[C1]](s64) + ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<6 x s32>) = G_BUILD_VECTOR [[EVEC]](s32), [[EVEC1]](s32), [[EVEC2]](s32), [[EVEC3]](s32), [[EVEC4]](s32), [[EVEC5]](s32) + ; CHECK-NEXT: RET_ReallyLR implicit [[BUILD_VECTOR]](<6 x s32>) + %0:_(<4 x s32>) = COPY $q0 + %1:_(<2 x s32>) = COPY $d0 + %2:_(s64) = G_CONSTANT i64 0 + %3:_(s64) = G_CONSTANT i64 1 + %4:_(s64) = G_CONSTANT i64 2 + %5:_(s64) = G_CONSTANT i64 3 + %10:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %2:_(s64) + %11:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %3:_(s64) + %12:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %4:_(s64) + %13:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %5:_(s64) + %14:_(s32) = G_EXTRACT_VECTOR_ELT %1:_(<2 x s32>), %2:_(s64) + %15:_(s32) = G_EXTRACT_VECTOR_ELT %1:_(<2 x s32>), %3:_(s64) + %18:_(<6 x s32>) = G_BUILD_VECTOR %10:_(s32), %11:_, %12:_, %13:_, %14:_, %15:_ + RET_ReallyLR implicit %18 +... +--- +name: concat_buildvector_shuffle_different_type +tracksRegLiveness: true +body: | + bb.1: + liveins: $q0, $q1 + ; CHECK-LABEL: name: concat_buildvector_shuffle_different_type + ; CHECK: liveins: $q0, $q1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<8 x s16>) = COPY $q1 + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1 + ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2 + ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 3 + ; CHECK-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](<4 x s32>), [[C]](s64) + ; CHECK-NEXT: [[EVEC1:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](<4 x s32>), [[C1]](s64) + ; CHECK-NEXT: [[EVEC2:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](<4 x s32>), [[C2]](s64) + ; CHECK-NEXT: [[EVEC3:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](<4 x s32>), [[C3]](s64) + ; CHECK-NEXT: [[EVEC4:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY1]](<8 x s16>), [[C]](s64) + ; CHECK-NEXT: [[EVEC5:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY1]](<8 x s16>), [[C1]](s64) + ; CHECK-NEXT: [[EVEC6:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY1]](<8 x s16>), [[C2]](s64) + ; CHECK-NEXT: [[EVEC7:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY1]](<8 x s16>), [[C3]](s64) + ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[EVEC]](s32), [[EVEC1]](s32), [[EVEC2]](s32), [[EVEC3]](s32), [[EVEC4]](s32), [[EVEC5]](s32), [[EVEC6]](s32), [[EVEC7]](s32) + ; CHECK-NEXT: RET_ReallyLR implicit [[BUILD_VECTOR]](<8 x s32>) + %0:_(<4 x s32>) = COPY $q0 + %1:_(<8 x s16>) = COPY $q1 + %2:_(s64) = G_CONSTANT i64 0 + %3:_(s64) = G_CONSTANT i64 1 + %4:_(s64) = G_CONSTANT i64 2 + %5:_(s64) = G_CONSTANT i64 3 + %10:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %2:_(s64) + %11:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %3:_(s64) + %12:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %4:_(s64) + %13:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %5:_(s64) + %14:_(s32) = G_EXTRACT_VECTOR_ELT %1:_(<8 x s16>), %2:_(s64) + %15:_(s32) = G_EXTRACT_VECTOR_ELT %1:_(<8 x s16>), %3:_(s64) + %16:_(s32) = G_EXTRACT_VECTOR_ELT %1:_(<8 x s16>), %4:_(s64) + %17:_(s32) = G_EXTRACT_VECTOR_ELT %1:_(<8 x s16>), %5:_(s64) + %18:_(<8 x s32>) = G_BUILD_VECTOR %10:_(s32), %11:_, %12:_, %13:_, %14:_, %15:_, %16:_, %17:_ + RET_ReallyLR implicit %18 +... +--- +name: concat_buildvector_shuffle_non_constant_id +tracksRegLiveness: true +body: | + bb.1: + liveins: $d0, $q0, $q1 + ; CHECK-LABEL: name: concat_buildvector_shuffle_non_constant_id + ; CHECK: liveins: $d0, $q0, $q1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1 + ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $d0 + ; CHECK-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](<4 x s32>), [[COPY2]](s64) + ; CHECK-NEXT: [[EVEC1:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY1]](<4 x s32>), [[COPY2]](s64) + ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[EVEC]](s32), [[EVEC1]](s32) + ; CHECK-NEXT: RET_ReallyLR implicit [[BUILD_VECTOR]](<2 x s32>) + %0:_(<4 x s32>) = COPY $q0 + %1:_(<4 x s32>) = COPY $q1 + %2:_(s64) = COPY $d0 + %10:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %2:_(s64) + %11:_(s32) = G_EXTRACT_VECTOR_ELT %1:_(<4 x s32>), %2:_(s64) + %12:_(<2 x s32>) = G_BUILD_VECTOR %10:_(s32), %11:_ + RET_ReallyLR implicit %12 +... +--- +name: concat_buildvector_shuffle_other_sources +tracksRegLiveness: true +body: | + bb.1: + liveins: $q0 + ; CHECK-LABEL: name: concat_buildvector_shuffle_other_sources + ; CHECK: liveins: $q0 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0 + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1 + ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 3 + ; CHECK-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](<4 x s32>), [[C]](s64) + ; CHECK-NEXT: [[EVEC1:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](<4 x s32>), [[C1]](s64) + ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 42 + ; CHECK-NEXT: [[EVEC2:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](<4 x s32>), [[C2]](s64) + ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[EVEC]](s32), [[EVEC1]](s32), [[C3]](s32), [[EVEC2]](s32) + ; CHECK-NEXT: RET_ReallyLR implicit [[BUILD_VECTOR]](<4 x s32>) + %0:_(<4 x s32>) = COPY $q0 + %2:_(s64) = G_CONSTANT i64 0 + %3:_(s64) = G_CONSTANT i64 1 + %4:_(s64) = G_CONSTANT i64 2 + %5:_(s64) = G_CONSTANT i64 3 + %10:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %2:_(s64) + %11:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %3:_(s64) + %12:_(s32) = G_CONSTANT i32 42 + %13:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %5:_(s64) + %18:_(<4 x s32>) = G_BUILD_VECTOR %10:_(s32), %11:_, %12:_, %13:_ + RET_ReallyLR implicit %18 +... +--- +name: reverse_concat_buildvector_shuffle_trunc +tracksRegLiveness: true +body: | + bb.1: + liveins: $q0, $q1 + ; CHECK-LABEL: name: reverse_concat_buildvector_shuffle_trunc + ; CHECK: liveins: $q0, $q1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1 + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1 + ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2 + ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 3 + ; CHECK-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](<4 x s32>), [[C]](s64) + ; CHECK-NEXT: [[EVEC1:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](<4 x s32>), [[C1]](s64) + ; CHECK-NEXT: [[EVEC2:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](<4 x s32>), [[C2]](s64) + ; CHECK-NEXT: [[EVEC3:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](<4 x s32>), [[C3]](s64) + ; CHECK-NEXT: [[EVEC4:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY1]](<4 x s32>), [[C]](s64) + ; CHECK-NEXT: [[EVEC5:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY1]](<4 x s32>), [[C1]](s64) + ; CHECK-NEXT: [[EVEC6:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY1]](<4 x s32>), [[C2]](s64) + ; CHECK-NEXT: [[EVEC7:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY1]](<4 x s32>), [[C3]](s64) + ; CHECK-NEXT: [[BUILD_VECTOR_TRUNC:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR_TRUNC [[EVEC]](s32), [[EVEC1]](s32), [[EVEC2]](s32), [[EVEC3]](s32), [[EVEC4]](s32), [[EVEC5]](s32), [[EVEC6]](s32), [[EVEC7]](s32) + ; CHECK-NEXT: RET_ReallyLR implicit [[BUILD_VECTOR_TRUNC]](<8 x s16>) + %0:_(<4 x s32>) = COPY $q0 + %1:_(<4 x s32>) = COPY $q1 + %2:_(s64) = G_CONSTANT i64 0 + %3:_(s64) = G_CONSTANT i64 1 + %4:_(s64) = G_CONSTANT i64 2 + %5:_(s64) = G_CONSTANT i64 3 + %10:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %2:_(s64) + %11:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %3:_(s64) + %12:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %4:_(s64) + %13:_(s32) = G_EXTRACT_VECTOR_ELT %0:_(<4 x s32>), %5:_(s64) + %14:_(s32) = G_EXTRACT_VECTOR_ELT %1:_(<4 x s32>), %2:_(s64) + %15:_(s32) = G_EXTRACT_VECTOR_ELT %1:_(<4 x s32>), %3:_(s64) + %16:_(s32) = G_EXTRACT_VECTOR_ELT %1:_(<4 x s32>), %4:_(s64) + %17:_(s32) = G_EXTRACT_VECTOR_ELT %1:_(<4 x s32>), %5:_(s64) + %18:_(<8 x s16>) = G_BUILD_VECTOR_TRUNC %10:_(s32), %11:_, %12:_, %13:_, %14:_, %15:_, %16:_, %17:_ + RET_ReallyLR implicit %18 +... diff --git a/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll b/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll index c0d91c1e0c836..98bd794b2d97e 100644 --- a/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll +++ b/llvm/test/CodeGen/AArch64/arm64-neon-copy.ll @@ -1923,49 +1923,12 @@ entry: } define <16 x i8> @test_concat_v16i8_v8i8_v8i8(<8 x i8> %x, <8 x i8> %y) #0 { -; CHECK-SD-LABEL: test_concat_v16i8_v8i8_v8i8: -; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q1 -; CHECK-SD-NEXT: mov v0.d[1], v1.d[0] -; CHECK-SD-NEXT: ret -; -; CHECK-GI-LABEL: test_concat_v16i8_v8i8_v8i8: -; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-GI-NEXT: mov b3, v0.b[1] -; CHECK-GI-NEXT: mov v2.b[0], v0.b[0] -; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q1 -; CHECK-GI-NEXT: mov b4, v0.b[2] -; CHECK-GI-NEXT: mov v2.b[1], v3.b[0] -; CHECK-GI-NEXT: mov b3, v0.b[3] -; CHECK-GI-NEXT: mov v2.b[2], v4.b[0] -; CHECK-GI-NEXT: mov b4, v0.b[4] -; CHECK-GI-NEXT: mov v2.b[3], v3.b[0] -; CHECK-GI-NEXT: mov b3, v0.b[5] -; CHECK-GI-NEXT: mov v2.b[4], v4.b[0] -; CHECK-GI-NEXT: mov b4, v0.b[6] -; CHECK-GI-NEXT: mov b0, v0.b[7] -; CHECK-GI-NEXT: mov v2.b[5], v3.b[0] -; CHECK-GI-NEXT: mov b3, v1.b[2] -; CHECK-GI-NEXT: mov v2.b[6], v4.b[0] -; CHECK-GI-NEXT: mov v2.b[7], v0.b[0] -; CHECK-GI-NEXT: mov b0, v1.b[1] -; CHECK-GI-NEXT: mov v2.b[8], v1.b[0] -; CHECK-GI-NEXT: mov v2.b[9], v0.b[0] -; CHECK-GI-NEXT: mov b0, v1.b[3] -; CHECK-GI-NEXT: mov v2.b[10], v3.b[0] -; CHECK-GI-NEXT: mov b3, v1.b[4] -; CHECK-GI-NEXT: mov v2.b[11], v0.b[0] -; CHECK-GI-NEXT: mov b0, v1.b[5] -; CHECK-GI-NEXT: mov v2.b[12], v3.b[0] -; CHECK-GI-NEXT: mov b3, v1.b[6] -; CHECK-GI-NEXT: mov v2.b[13], v0.b[0] -; CHECK-GI-NEXT: mov b0, v1.b[7] -; CHECK-GI-NEXT: mov v2.b[14], v3.b[0] -; CHECK-GI-NEXT: mov v2.b[15], v0.b[0] -; CHECK-GI-NEXT: mov v0.16b, v2.16b -; CHECK-GI-NEXT: ret +; CHECK-LABEL: test_concat_v16i8_v8i8_v8i8: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1 +; CHECK-NEXT: mov v0.d[1], v1.d[0] +; CHECK-NEXT: ret entry: %vecext = extractelement <8 x i8> %x, i32 0 %vecinit = insertelement <16 x i8> undef, i8 %vecext, i32 0 @@ -2094,27 +2057,12 @@ entry: } define <8 x i16> @test_concat_v8i16_v4i16_v4i16(<4 x i16> %x, <4 x i16> %y) #0 { -; CHECK-SD-LABEL: test_concat_v8i16_v4i16_v4i16: -; CHECK-SD: // %bb.0: // %entry -; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q1 -; CHECK-SD-NEXT: mov v0.d[1], v1.d[0] -; CHECK-SD-NEXT: ret -; -; CHECK-GI-LABEL: test_concat_v8i16_v4i16_v4i16: -; CHECK-GI: // %bb.0: // %entry -; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-GI-NEXT: mov v2.h[0], v0.h[0] -; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q1 -; CHECK-GI-NEXT: mov v2.h[1], v0.h[1] -; CHECK-GI-NEXT: mov v2.h[2], v0.h[2] -; CHECK-GI-NEXT: mov v2.h[3], v0.h[3] -; CHECK-GI-NEXT: mov v2.h[4], v1.h[0] -; CHECK-GI-NEXT: mov v2.h[5], v1.h[1] -; CHECK-GI-NEXT: mov v2.h[6], v1.h[2] -; CHECK-GI-NEXT: mov v2.h[7], v1.h[3] -; CHECK-GI-NEXT: mov v0.16b, v2.16b -; CHECK-GI-NEXT: ret +; CHECK-LABEL: test_concat_v8i16_v4i16_v4i16: +; CHECK: // %bb.0: // %entry +; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1 +; CHECK-NEXT: mov v0.d[1], v1.d[0] +; CHECK-NEXT: ret entry: %vecext = extractelement <4 x i16> %x, i32 0 %vecinit = insertelement <8 x i16> undef, i16 %vecext, i32 0