Merge branch 'main' into users/arsenm/amdgpu/test-vgpr-cd-selection-mfma-xf32

Author: arsenm

clang/lib/Sema/SemaModule.cpp

@@ -13,7 +13,7 @@
 
 #include "clang/AST/ASTConsumer.h"
 #include "clang/AST/ASTMutationListener.h"
-#include "clang/AST/RecursiveASTVisitor.h"
+#include "clang/AST/DynamicRecursiveASTVisitor.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Sema/ParsedAttr.h"
@@ -1422,14 +1422,14 @@ bool ExposureChecker::checkExposure(const CXXRecordDecl *RD, bool Diag) {
   return IsExposure;
 }
 
-template <typename CallbackTy>
-class ReferenceTULocalChecker
-    : public clang::RecursiveASTVisitor<ReferenceTULocalChecker<CallbackTy>> {
+class ReferenceTULocalChecker : public DynamicRecursiveASTVisitor {
 public:
+  using CallbackTy = std::function<void(DeclRefExpr *, ValueDecl *)>;
+
   ReferenceTULocalChecker(ExposureChecker &C, CallbackTy &&Callback)
       : Checker(C), Callback(std::move(Callback)) {}
 
-  bool VisitDeclRefExpr(DeclRefExpr *DRE) {
+  bool VisitDeclRefExpr(DeclRefExpr *DRE) override {
     ValueDecl *Referenced = DRE->getDecl();
     if (!Referenced)
       return true;
@@ -1468,10 +1468,6 @@ class ReferenceTULocalChecker
   CallbackTy Callback;
 };
 
-template <typename CallbackTy>
-ReferenceTULocalChecker(ExposureChecker &, CallbackTy &&)
-    -> ReferenceTULocalChecker<CallbackTy>;
-
 bool ExposureChecker::checkExposure(const Stmt *S, bool Diag) {
   if (!S)
     return false;

llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

@@ -2244,6 +2244,9 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
 
   // Try to optimize header mask recipes away to their EVL variants.
   for (VPValue *HeaderMask : collectAllHeaderMasks(Plan)) {
+    // TODO: Split optimizeMaskToEVL out and move into
+    // VPlanTransforms::optimize. transformRecipestoEVLRecipes should be run in
+    // tryToBuildVPlanWithVPRecipes beforehand.
     for (VPUser *U : collectUsersRecursively(HeaderMask)) {
       auto *CurRecipe = cast<VPRecipeBase>(U);
       VPRecipeBase *EVLRecipe =
@@ -2265,6 +2268,20 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
       }
       ToErase.push_back(CurRecipe);
     }
+
+    // Replace header masks with a mask equivalent to predicating by EVL:
+    //
+    // icmp ule widen-canonical-iv backedge-taken-count
+    // ->
+    // icmp ult step-vector, EVL
+    VPRecipeBase *EVLR = EVL.getDefiningRecipe();
+    VPBuilder Builder(EVLR->getParent(), std::next(EVLR->getIterator()));
+    Type *EVLType = TypeInfo.inferScalarType(&EVL);
+    VPValue *EVLMask = Builder.createICmp(
+        CmpInst::ICMP_ULT,
+        Builder.createNaryOp(VPInstruction::StepVector, {}, EVLType), &EVL);
+    HeaderMask->replaceAllUsesWith(EVLMask);
+    ToErase.push_back(HeaderMask->getDefiningRecipe());
   }
 
   for (VPRecipeBase *R : reverse(ToErase)) {

llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp

@@ -171,7 +171,8 @@ bool VPlanVerifier::verifyEVLRecipe(const VPInstruction &EVL) const {
         .Case<VPInstructionWithType>(
             [&](const VPInstructionWithType *S) { return VerifyEVLUse(*S, 0); })
         .Case<VPInstruction>([&](const VPInstruction *I) {
-          if (I->getOpcode() == Instruction::PHI)
+          if (I->getOpcode() == Instruction::PHI ||
+              I->getOpcode() == Instruction::ICmp)
             return VerifyEVLUse(*I, 1);
           switch (I->getOpcode()) {
           case Instruction::Add: