Make this a non-backend specific optimisation & check all unmerged values are used

HolyMolyCowMan · HolyMolyCowMan · commit 69ba5ab26d6b · 2025-11-03T10:14:08.000Z
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -647,6 +647,12 @@ class CombinerHelper {
   bool matchRotateOutOfRange(MachineInstr &MI) const;
   void applyRotateOutOfRange(MachineInstr &MI) const;
 
+  bool matchCombineBuildUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,
+                                Register &UnmergeSrc) const;
+  void applyCombineBuildUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,
+                                MachineIRBuilder &B,
+                                Register &UnmergeSrc) const;
+
   bool matchUseVectorTruncate(MachineInstr &MI, Register &MatchInfo) const;
   void applyUseVectorTruncate(MachineInstr &MI, Register &MatchInfo) const;
 
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -921,6 +921,15 @@ def merge_of_x_and_zero : GICombineRule <
          [{ return Helper.matchMergeXAndZero(*${MI}, ${matchinfo}); }]),
   (apply [{ Helper.applyBuildFn(*${MI}, ${matchinfo}); }])>;
 
+// Transform build_vector(unmerge(src, 0), ... unmerge(src, n), undef, ..., undef)
+// => concat_vectors(src, undef)
+def combine_build_unmerge : GICombineRule<
+  (defs root:$root, register_matchinfo:$unmergeSrc),
+  (match (G_BUILD_VECTOR $dst, GIVariadic<>:$unused):$root,
+         [{ return Helper.matchCombineBuildUnmerge(*${root}, MRI, ${unmergeSrc}); }]),
+  (apply [{ Helper.applyCombineBuildUnmerge(*${root}, MRI, B, ${unmergeSrc}); }])
+>;
+
 def merge_combines: GICombineGroup<[
   unmerge_anyext_build_vector,
   unmerge_merge,
@@ -930,7 +939,8 @@ def merge_combines: GICombineGroup<[
   unmerge_dead_to_trunc,
   unmerge_zext_to_zext,
   merge_of_x_and_undef,
-  merge_of_x_and_zero
+  merge_of_x_and_zero,
+  combine_build_unmerge
 ]>;
 
 // Under certain conditions, transform:
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -3463,6 +3463,91 @@ static bool isConstValidTrue(const TargetLowering &TLI, unsigned ScalarSizeBits,
          isConstTrueVal(TLI, Cst, IsVector, IsFP);
 }
 
+// This pattern aims to match the following shape to avoid extra mov
+// instructions
+// G_BUILD_VECTOR(
+//   G_UNMERGE_VALUES(src, 0)
+//   G_UNMERGE_VALUES(src, 1)
+//   G_IMPLICIT_DEF
+//   G_IMPLICIT_DEF
+// )
+// ->
+// G_CONCAT_VECTORS(
+//   src,
+//   undef
+// )
+bool CombinerHelper::matchCombineBuildUnmerge(MachineInstr &MI,
+                                              MachineRegisterInfo &MRI,
+                                              Register &UnmergeSrc) const {
+  assert(MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
+
+  unsigned BuildUseCount = MI.getNumOperands() - 1;
+
+  if (BuildUseCount % 2 != 0)
+    return false;
+
+  unsigned NumUnmerge = BuildUseCount / 2;
+
+  // Check the first operand is an unmerge
+  auto *MaybeUnmerge = getDefIgnoringCopies(MI.getOperand(1).getReg(), MRI);
+  if (MaybeUnmerge->getOpcode() != TargetOpcode::G_UNMERGE_VALUES)
+    return false;
+
+  // Check that the resultant concat will be legal
+  auto UnmergeEltSize =
+      MRI.getType(MaybeUnmerge->getOperand(1).getReg()).getScalarSizeInBits();
+  auto UnmergeEltCount = MaybeUnmerge->getNumDefs();
+
+  if (UnmergeEltCount < 2 || (UnmergeEltSize * UnmergeEltCount != 64 &&
+                              UnmergeEltSize * UnmergeEltCount != 128))
+    return false;
+
+  // Check that all of the operands before the midpoint come from the same
+  // unmerge and are in the same order as they are used in the build_vector
+  for (unsigned I = 0; I < NumUnmerge; ++I) {
+    auto MaybeUnmergeReg = MI.getOperand(I + 1).getReg();
+    auto *Unmerge = getDefIgnoringCopies(MaybeUnmergeReg, MRI);
+
+    if (Unmerge != MaybeUnmerge)
+      return false;
+
+    if (Unmerge->getOperand(I).getReg() != MaybeUnmergeReg)
+      return false;
+  }
+
+  // Check that all of the unmerged values are used
+  if (UnmergeEltCount != NumUnmerge)
+    return false;
+
+  // Check that all of the operands after the mid point are undefs.
+  for (unsigned I = NumUnmerge; I < BuildUseCount; ++I) {
+    auto *Undef = getDefIgnoringCopies(MI.getOperand(I + 1).getReg(), MRI);
+
+    if (Undef->getOpcode() != TargetOpcode::G_IMPLICIT_DEF)
+      return false;
+  }
+
+  // Unmerge should only use one register so we can use the last one
+  for (auto &UnmergeUse :
+       getDefIgnoringCopies(MI.getOperand(1).getReg(), MRI)->all_uses())
+    UnmergeSrc = UnmergeUse.getReg();
+
+  return true;
+}
+
+void CombinerHelper::applyCombineBuildUnmerge(MachineInstr &MI,
+                                              MachineRegisterInfo &MRI,
+                                              MachineIRBuilder &B,
+                                              Register &UnmergeSrc) const {
+  assert(UnmergeSrc && "Expected there to be one matching G_UNMERGE_VALUES");
+  B.setInstrAndDebugLoc(MI);
+
+  Register UndefVec = B.buildUndef(MRI.getType(UnmergeSrc)).getReg(0);
+  B.buildConcatVectors(MI.getOperand(0), {UnmergeSrc, UndefVec});
+
+  MI.eraseFromParent();
+}
+
 // This combine tries to reduce the number of scalarised G_TRUNC instructions by
 // using vector truncates instead
 //
diff --git a/llvm/lib/Target/AArch64/AArch64Combine.td b/llvm/lib/Target/AArch64/AArch64Combine.td
@@ -332,13 +332,6 @@ def combine_mul_cmlt : GICombineRule<
   (apply [{ applyCombineMulCMLT(*${root}, MRI, B, ${matchinfo}); }])
 >;
 
-def combine_build_unmerge : GICombineRule<
-  (defs root:$root, register_matchinfo:$unmergeSrc),
-  (match (G_BUILD_VECTOR $dst, GIVariadic<>:$unused):$root,
-         [{ return matchCombineBuildUnmerge(*${root}, MRI, ${unmergeSrc}); }]),
-  (apply [{ applyCombineBuildUnmerge(*${root}, MRI, B, ${unmergeSrc}); }])
->;
-
 // Post-legalization combines which should happen at all optimization levels.
 // (E.g. ones that facilitate matching for the selector) For example, matching
 // pseudos.
@@ -373,7 +366,7 @@ def AArch64PostLegalizerCombiner
                         select_to_minmax, or_to_bsp, combine_concat_vector,
                         commute_constant_to_rhs, extract_vec_elt_combines,
                         push_freeze_to_prevent_poison_from_propagating,
-                        combine_mul_cmlt, combine_use_vector_truncate, combine_build_unmerge,
+                        combine_mul_cmlt, combine_use_vector_truncate,
                         extmultomull, truncsat_combines, lshr_of_trunc_of_lshr,
                         funnel_shift_from_or_shift_constants_are_legal]> {
 }
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/GISel/AArch64PostLegalizerCombiner.cpp
@@ -134,85 +134,6 @@ bool isZeroExtended(Register R, MachineRegisterInfo &MRI) {
   return MRI.getVRegDef(R)->getOpcode() == TargetOpcode::G_ZEXT;
 }
 
-// This pattern aims to match the following shape to avoid extra mov
-// instructions
-// G_BUILD_VECTOR(
-//   G_UNMERGE_VALUES(src, 0)
-//   G_UNMERGE_VALUES(src, 1)
-//   G_IMPLICIT_DEF
-//   G_IMPLICIT_DEF
-// )
-// ->
-// G_CONCAT_VECTORS(
-//   src,
-//   undef
-// )
-bool matchCombineBuildUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,
-                              Register &UnmergeSrc) {
-  assert(MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
-
-  unsigned BuildUseCount = MI.getNumOperands() - 1;
-
-  if (BuildUseCount % 2 != 0)
-    return false;
-
-  unsigned HalfWayIndex = BuildUseCount / 2;
-
-  // Check the first operand is an unmerge
-  auto *MaybeUnmerge = getDefIgnoringCopies(MI.getOperand(1).getReg(), MRI);
-  if (MaybeUnmerge->getOpcode() != TargetOpcode::G_UNMERGE_VALUES)
-    return false;
-
-  // Check that the resultant concat will be legal
-  auto UnmergeEltSize =
-      MRI.getType(MaybeUnmerge->getOperand(1).getReg()).getScalarSizeInBits();
-  auto UnmergeEltCount = MaybeUnmerge->getNumDefs();
-
-  if (UnmergeEltCount < 2 || (UnmergeEltSize * UnmergeEltCount != 64 &&
-                              UnmergeEltSize * UnmergeEltCount != 128))
-    return false;
-
-  // Check that all of the operands before the midpoint come from the same
-  // unmerge and are in the same order as they are used in the build_vector
-  for (unsigned I = 0; I < HalfWayIndex; ++I) {
-    auto MaybeUnmergeReg = MI.getOperand(I + 1).getReg();
-    auto *Unmerge = getDefIgnoringCopies(MaybeUnmergeReg, MRI);
-
-    if (Unmerge != MaybeUnmerge)
-      return false;
-
-    if (Unmerge->getOperand(I).getReg() != MaybeUnmergeReg)
-      return false;
-  }
-
-  // Check that all of the operands after the mid point are undefs.
-  for (unsigned I = HalfWayIndex; I < BuildUseCount; ++I) {
-    auto *Undef = getDefIgnoringCopies(MI.getOperand(I + 1).getReg(), MRI);
-
-    if (Undef->getOpcode() != TargetOpcode::G_IMPLICIT_DEF)
-      return false;
-  }
-
-  // Unmerge should only use one register so we can use the last one
-  for (auto &UnmergeUse :
-       getDefIgnoringCopies(MI.getOperand(1).getReg(), MRI)->all_uses())
-    UnmergeSrc = UnmergeUse.getReg();
-
-  return true;
-}
-
-void applyCombineBuildUnmerge(MachineInstr &MI, MachineRegisterInfo &MRI,
-                              MachineIRBuilder &B, Register &UnmergeSrc) {
-  assert(UnmergeSrc && "Expected there to be one matching G_UNMERGE_VALUES");
-  B.setInstrAndDebugLoc(MI);
-
-  Register UndefVec = MRI.createGenericVirtualRegister(MRI.getType(UnmergeSrc));
-  B.buildUndef(UndefVec);
-  B.buildConcatVectors(MI.getOperand(0), {UnmergeSrc, UndefVec});
-
-  MI.eraseFromParent();
-}
-
 bool matchAArch64MulConstCombine(
     MachineInstr &MI, MachineRegisterInfo &MRI,
     std::function<void(MachineIRBuilder &B, Register DstReg)> &ApplyFn) {
