[OpenACC] Partial Reduction recipe Lowering (#155635)

This patch implements basic reduction recipe lowering, plus adds a bunch
of tests for it that should be meaningful later. At the moment, all this
does is ensure that we get the init 'alloca' set right (the actual
initializer isn't done correctly yet, and will be in a followup), an
empty combiner (though the type of certain operations probably has to be
different as well, when we get to those), and a full-destruction, as we
already have the infrastructure for it.

Author: erichkeane

clang/lib/CIR/CodeGen/CIRGenOpenACCClause.cpp

@@ -357,7 +357,8 @@ class OpenACCClauseCIREmitter final
   }
 
   template <typename RecipeTy>
-  std::string getRecipeName(SourceRange loc, QualType baseType) {
+  std::string getRecipeName(SourceRange loc, QualType baseType,
+                            OpenACCReductionOperator reductionOp) {
     std::string recipeName;
     {
       llvm::raw_string_ostream stream(recipeName);
@@ -371,12 +372,40 @@ class OpenACCClauseCIREmitter final
       } else if constexpr (std::is_same_v<RecipeTy,
                                           mlir::acc::ReductionRecipeOp>) {
         stream << "reduction_";
-        // TODO: OpenACC: once we have this part implemented, we can remove the
-        // SourceRange `loc` variable from this function. We don't have the
-        // reduction operation here well enough to know how to spell this
-        // correctly (+ == 'add', etc), so when we implement 'reduction' we have
-        // to do that here.
-        cgf.cgm.errorNYI(loc, "OpenACC reduction recipe name creation");
+        // Values here are a little weird (for bitwise and/or is 'i' prefix, and
+        // logical ops with 'l'), but are chosen to be the same as the MLIR
+        // dialect names as well as to match the Flang versions of these.
+        switch (reductionOp) {
+        case OpenACCReductionOperator::Addition:
+          stream << "add_";
+          break;
+        case OpenACCReductionOperator::Multiplication:
+          stream << "mul_";
+          break;
+        case OpenACCReductionOperator::Max:
+          stream << "max_";
+          break;
+        case OpenACCReductionOperator::Min:
+          stream << "min_";
+          break;
+        case OpenACCReductionOperator::BitwiseAnd:
+          stream << "iand_";
+          break;
+        case OpenACCReductionOperator::BitwiseOr:
+          stream << "ior_";
+          break;
+        case OpenACCReductionOperator::BitwiseXOr:
+          stream << "xor_";
+          break;
+        case OpenACCReductionOperator::And:
+          stream << "land_";
+          break;
+        case OpenACCReductionOperator::Or:
+          stream << "lor_";
+          break;
+        case OpenACCReductionOperator::Invalid:
+          llvm_unreachable("invalid reduction operator");
+        }
       } else {
         static_assert(!sizeof(RecipeTy), "Unknown Recipe op kind");
       }
@@ -419,7 +448,9 @@ class OpenACCClauseCIREmitter final
   }
 
   // Create the 'init' section of the recipe, including the 'copy' section for
-  // 'firstprivate'.
+  // 'firstprivate'.  Note that this function is not 'insertion point' clean, in
+  // that it alters the insertion point to be inside of the 'destroy' section of
+  // the recipe, but doesn't restore it aftewards.
   template <typename RecipeTy>
   void createRecipeInitCopy(mlir::Location loc, mlir::Location locEnd,
                             SourceRange exprRange, mlir::Value mainOp,
@@ -485,6 +516,27 @@ class OpenACCClauseCIREmitter final
     }
   }
 
+  // This function generates the 'combiner' section for a reduction recipe. Note
+  // that this function is not 'insertion point' clean, in that it alters the
+  // insertion point to be inside of the 'combiner' section of the recipe, but
+  // doesn't restore it aftewards.
+  void createReductionRecipeCombiner(mlir::Location loc, mlir::Location locEnd,
+                                     mlir::Value mainOp,
+                                     mlir::acc::ReductionRecipeOp recipe) {
+    mlir::Block *block = builder.createBlock(
+        &recipe.getCombinerRegion(), recipe.getCombinerRegion().end(),
+        {mainOp.getType(), mainOp.getType()}, {loc, loc});
+    builder.setInsertionPointToEnd(&recipe.getCombinerRegion().back());
+
+    mlir::BlockArgument lhsArg = block->getArgument(0);
+
+    mlir::acc::YieldOp::create(builder, locEnd, lhsArg);
+  }
+
+  // This function generates the 'destroy' section for a recipe. Note
+  // that this function is not 'insertion point' clean, in that it alters the
+  // insertion point to be inside of the 'destroy' section of the recipe, but
+  // doesn't restore it aftewards.
   void createRecipeDestroySection(mlir::Location loc, mlir::Location locEnd,
                                   mlir::Value mainOp, CharUnits alignment,
                                   QualType baseType,
@@ -502,30 +554,68 @@ class OpenACCClauseCIREmitter final
     mlir::acc::YieldOp::create(builder, locEnd);
   }
 
+  mlir::acc::ReductionOperator convertReductionOp(OpenACCReductionOperator op) {
+    switch (op) {
+    case OpenACCReductionOperator::Addition:
+      return mlir::acc::ReductionOperator::AccAdd;
+    case OpenACCReductionOperator::Multiplication:
+      return mlir::acc::ReductionOperator::AccMul;
+    case OpenACCReductionOperator::Max:
+      return mlir::acc::ReductionOperator::AccMax;
+    case OpenACCReductionOperator::Min:
+      return mlir::acc::ReductionOperator::AccMin;
+    case OpenACCReductionOperator::BitwiseAnd:
+      return mlir::acc::ReductionOperator::AccIand;
+    case OpenACCReductionOperator::BitwiseOr:
+      return mlir::acc::ReductionOperator::AccIor;
+    case OpenACCReductionOperator::BitwiseXOr:
+      return mlir::acc::ReductionOperator::AccXor;
+    case OpenACCReductionOperator::And:
+      return mlir::acc::ReductionOperator::AccLand;
+    case OpenACCReductionOperator::Or:
+      return mlir::acc::ReductionOperator::AccLor;
+    case OpenACCReductionOperator::Invalid:
+      llvm_unreachable("invalid reduction operator");
+    }
+
+    llvm_unreachable("invalid reduction operator");
+  }
+
   template <typename RecipeTy>
   RecipeTy getOrCreateRecipe(ASTContext &astCtx, const Expr *varRef,
                              const VarDecl *varRecipe, const VarDecl *temporary,
+                             OpenACCReductionOperator reductionOp,
                              DeclContext *dc, QualType baseType,
                              mlir::Value mainOp) {
     mlir::ModuleOp mod = builder.getBlock()
                              ->getParent()
                              ->template getParentOfType<mlir::ModuleOp>();
 
-    std::string recipeName =
-        getRecipeName<RecipeTy>(varRef->getSourceRange(), baseType);
+    std::string recipeName = getRecipeName<RecipeTy>(varRef->getSourceRange(),
+                                                     baseType, reductionOp);
     if (auto recipe = mod.lookupSymbol<RecipeTy>(recipeName))
       return recipe;
 
     mlir::Location loc = cgf.cgm.getLoc(varRef->getBeginLoc());
     mlir::Location locEnd = cgf.cgm.getLoc(varRef->getEndLoc());
 
     mlir::OpBuilder modBuilder(mod.getBodyRegion());
-    auto recipe =
-        RecipeTy::create(modBuilder, loc, recipeName, mainOp.getType());
+    RecipeTy recipe;
+
+    if constexpr (std::is_same_v<RecipeTy, mlir::acc::ReductionRecipeOp>) {
+      recipe = RecipeTy::create(modBuilder, loc, recipeName, mainOp.getType(),
+                                convertReductionOp(reductionOp));
+    } else {
+      recipe = RecipeTy::create(modBuilder, loc, recipeName, mainOp.getType());
+    }
 
     createRecipeInitCopy(loc, locEnd, varRef->getSourceRange(), mainOp, recipe,
                          varRecipe, temporary);
 
+    if constexpr (std::is_same_v<RecipeTy, mlir::acc::ReductionRecipeOp>) {
+      createReductionRecipeCombiner(loc, locEnd, mainOp, recipe);
+    }
+
     if (varRecipe && varRecipe->needsDestruction(cgf.getContext()))
       createRecipeDestroySection(loc, locEnd, mainOp,
                                  cgf.getContext().getDeclAlign(varRecipe),
@@ -1166,6 +1256,8 @@ class OpenACCClauseCIREmitter final
           mlir::OpBuilder::InsertionGuard guardCase(builder);
           auto recipe = getOrCreateRecipe<mlir::acc::PrivateRecipeOp>(
               cgf.getContext(), varExpr, varRecipe, /*temporary=*/nullptr,
+              OpenACCReductionOperator::Invalid,
+
               Decl::castToDeclContext(cgf.curFuncDecl), opInfo.baseType,
               privateOp.getResult());
           // TODO: OpenACC: The dialect is going to change in the near future to
@@ -1200,7 +1292,7 @@ class OpenACCClauseCIREmitter final
           mlir::OpBuilder::InsertionGuard guardCase(builder);
           auto recipe = getOrCreateRecipe<mlir::acc::FirstprivateRecipeOp>(
               cgf.getContext(), varExpr, varRecipe.RecipeDecl,
-              varRecipe.InitFromTemporary,
+              varRecipe.InitFromTemporary, OpenACCReductionOperator::Invalid,
               Decl::castToDeclContext(cgf.curFuncDecl), opInfo.baseType,
               firstPrivateOp.getResult());
 
@@ -1219,6 +1311,40 @@ class OpenACCClauseCIREmitter final
       llvm_unreachable("Unknown construct kind in VisitFirstPrivateClause");
     }
   }
+
+  void VisitReductionClause(const OpenACCReductionClause &clause) {
+    if constexpr (isOneOfTypes<OpTy, mlir::acc::ParallelOp, mlir::acc::SerialOp,
+                               mlir::acc::LoopOp>) {
+      for (const auto [varExpr, varRecipe] :
+           llvm::zip_equal(clause.getVarList(), clause.getRecipes())) {
+        CIRGenFunction::OpenACCDataOperandInfo opInfo =
+            cgf.getOpenACCDataOperandInfo(varExpr);
+
+        auto reductionOp = mlir::acc::ReductionOp::create(
+            builder, opInfo.beginLoc, opInfo.varValue, /*structured=*/true,
+            /*implicit=*/false, opInfo.name, opInfo.bounds);
+        reductionOp.setDataClause(mlir::acc::DataClause::acc_reduction);
+
+        {
+          mlir::OpBuilder::InsertionGuard guardCase(builder);
+
+          auto recipe = getOrCreateRecipe<mlir::acc::ReductionRecipeOp>(
+              cgf.getContext(), varExpr, varRecipe.RecipeDecl,
+              /*temporary=*/nullptr, clause.getReductionOp(),
+              Decl::castToDeclContext(cgf.curFuncDecl), opInfo.baseType,
+              reductionOp.getResult());
+
+          operation.addReduction(builder.getContext(), reductionOp, recipe);
+        }
+      }
+    } else if constexpr (isCombinedType<OpTy>) {
+      // Despite this being valid on ParallelOp or SerialOp, combined type
+      // applies to the 'loop'.
+      applyToLoopOp(clause);
+    } else {
+      llvm_unreachable("Unknown construct kind in VisitReductionClause");
+    }
+  }
 };
 
 template <typename OpTy>