From bac893fc0b403f330ee2781b3f8e557bf871e263 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Fri, 11 Jul 2025 19:54:49 +0100
Subject: [PATCH 1/3] [VPlan] Move out canNarrowOps (NFC).

---
 .../Transforms/Vectorize/VPlanTransforms.cpp  | 42 ++++++++++++-------
 1 file changed, 27 insertions(+), 15 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 25557f1d5d651..cf1bbbb1c9a72 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4268,6 +4268,32 @@ static bool canNarrowLoad(VPWidenRecipe *WideMember0, unsigned OpIdx,
   return false;
 }
 
+static bool canNarrowOps(ArrayRef<VPValue *> Ops) {
+  SmallVector<VPValue *> Ops0;
+  auto *WideMember0 = dyn_cast<VPWidenRecipe>(Ops[0]);
+  if (!WideMember0)
+    return false;
+
+  for (unsigned Idx = 0; Idx != WideMember0->getNumOperands(); ++Idx) {
+    SmallVector<VPValue *> Ops0;
+    for (const auto &[I, V] : enumerate(Ops)) {
+      auto *R = dyn_cast_or_null<VPWidenRecipe>(V);
+      if (!R || R->getOpcode() != WideMember0->getOpcode() ||
+          R->getNumOperands() > 2)
+        return false;
+
+      Ops0.push_back(R->getOperand(Idx));
+    }
+    if (any_of(enumerate(Ops0), [WideMember0, Idx](const auto &P) {
+          const auto &[OpIdx, OpV] = P;
+          return !canNarrowLoad(WideMember0, Idx, OpV, OpIdx);
+        }))
+      return false;
+  }
+
+  return true;
+}
+
 /// Returns true if \p IR is a full interleave group with factor and number of
 /// members both equal to \p VF. The interleave group must also access the full
 /// vector width \p VectorRegWidth.
@@ -4441,22 +4467,8 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
 
     // Check if all values feeding InterleaveR are matching wide recipes, which
     // operands that can be narrowed.
-    auto *WideMember0 =
-        dyn_cast_or_null<VPWidenRecipe>(InterleaveR->getStoredValues()[0]);
-    if (!WideMember0)
+    if (!canNarrowOps(InterleaveR->getStoredValues()))
       return;
-    for (const auto &[I, V] : enumerate(InterleaveR->getStoredValues())) {
-      auto *R = dyn_cast_or_null<VPWidenRecipe>(V);
-      if (!R || R->getOpcode() != WideMember0->getOpcode() ||
-          R->getNumOperands() > 2)
-        return;
-      if (any_of(enumerate(R->operands()),
-                 [WideMember0, Idx = I](const auto &P) {
-                   const auto &[OpIdx, OpV] = P;
-                   return !canNarrowLoad(WideMember0, OpIdx, OpV, Idx);
-                 }))
-        return;
-    }
     StoreGroups.push_back(InterleaveR);
   }
 

From 471e12bd44a068175978c7d9caae22d632682f41 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Mon, 10 Nov 2025 17:25:15 +0000
Subject: [PATCH 2/3] !fixup restructure op handling a bit

---
 .../Transforms/Vectorize/VPlanTransforms.cpp    | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index cf1bbbb1c9a72..853605f103ab9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4274,16 +4274,17 @@ static bool canNarrowOps(ArrayRef<VPValue *> Ops) {
   if (!WideMember0)
     return false;
 
+  for (const auto &[_, V] : enumerate(Ops)) {
+    auto *R = dyn_cast<VPWidenRecipe>(V);
+    if (!R || R->getOpcode() != WideMember0->getOpcode() ||
+        R->getNumOperands() > 2)
+      return false;
+  }
+
   for (unsigned Idx = 0; Idx != WideMember0->getNumOperands(); ++Idx) {
     SmallVector<VPValue *> Ops0;
-    for (const auto &[I, V] : enumerate(Ops)) {
-      auto *R = dyn_cast_or_null<VPWidenRecipe>(V);
-      if (!R || R->getOpcode() != WideMember0->getOpcode() ||
-          R->getNumOperands() > 2)
-        return false;
-
-      Ops0.push_back(R->getOperand(Idx));
-    }
+    for (VPValue *Op : Ops)
+      Ops0.push_back(Op->getDefiningRecipe()->getOperand(Idx));
     if (any_of(enumerate(Ops0), [WideMember0, Idx](const auto &P) {
           const auto &[OpIdx, OpV] = P;
           return !canNarrowLoad(WideMember0, Idx, OpV, OpIdx);

From be3ecf289356ac4c38f5d6dddb272a0812430ebc Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Wed, 19 Nov 2025 22:53:28 +0000
Subject: [PATCH 3/3] !fixup Ops0 -> OpsI, thanks!

---
 llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 853605f103ab9..fe5652bd85bda 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -4282,10 +4282,11 @@ static bool canNarrowOps(ArrayRef<VPValue *> Ops) {
   }
 
   for (unsigned Idx = 0; Idx != WideMember0->getNumOperands(); ++Idx) {
-    SmallVector<VPValue *> Ops0;
+    SmallVector<VPValue *> OpsI;
     for (VPValue *Op : Ops)
-      Ops0.push_back(Op->getDefiningRecipe()->getOperand(Idx));
-    if (any_of(enumerate(Ops0), [WideMember0, Idx](const auto &P) {
+      OpsI.push_back(Op->getDefiningRecipe()->getOperand(Idx));
+
+    if (any_of(enumerate(OpsI), [WideMember0, Idx](const auto &P) {
           const auto &[OpIdx, OpV] = P;
           return !canNarrowLoad(WideMember0, Idx, OpV, OpIdx);
         }))