[WIP][llvm] Flag to forcibly outline fixed point mul/div intrinsics

clang/include/clang/Basic/CodeGenOptions.def

@@ -483,6 +483,9 @@ CODEGENOPT(StaticClosure, 1, 0)
 /// Assume that UAVs/SRVs may alias
 CODEGENOPT(ResMayAlias, 1, 0)
 
+/// Outline fixed point multiplication and division intrinsics.
+CODEGENOPT(OutlineFixedPointMulDiv, 1, 0)
+
 /// Controls how unwind v2 (epilog) information should be generated for x64
 /// Windows.
 ENUM_CODEGENOPT(WinX64EHUnwindV2, llvm::WinX64EHUnwindV2Mode,

clang/include/clang/Driver/Options.td

@@ -2741,6 +2741,15 @@ defm strict_float_cast_overflow : BoolFOption<"strict-float-cast-overflow",
             " of the target's native float-to-int conversion instructions">,
   PosFlag<SetTrue, [], [ClangOption], "Assume that overflowing float-to-int casts are undefined (default)">>;
 
+defm outline_fixed_point_mul_div_intrinsics
+    : BoolFOption<
+          "outline-fixed-point-mul-div-intrinsics",
+          CodeGenOpts<"OutlineFixedPointMulDiv">, DefaultFalse,
+          NegFlag<SetFalse, [], [ClangOption, CC1Option], "Do not outline ">,
+          PosFlag<SetTrue, [], [ClangOption, CC1Option], "Outline ">,
+          BothFlags<[], [ClangOption, CC1Option],
+                    "the fixed point multiplication and division intrinsics">>;
+
 defm protect_parens : BoolFOption<"protect-parens",
   LangOpts<"ProtectParens">, DefaultFalse,
   PosFlag<SetTrue, [], [ClangOption, CLOption, CC1Option],

clang/lib/CodeGen/CGExprScalar.cpp

@@ -4543,11 +4543,15 @@ Value *ScalarExprEmitter::EmitFixedPointBinOp(const BinOpInfo &op) {
     break;
   case BO_MulAssign:
   case BO_Mul:
-    Result = FPBuilder.CreateMul(LHS, LHSFixedSema, RHS, RHSFixedSema);
+    Result =
+        FPBuilder.CreateMul(LHS, LHSFixedSema, RHS, RHSFixedSema,
+                            CGF.CGM.getCodeGenOpts().OutlineFixedPointMulDiv);
     break;
   case BO_DivAssign:
   case BO_Div:
-    Result = FPBuilder.CreateDiv(LHS, LHSFixedSema, RHS, RHSFixedSema);
+    Result =
+        FPBuilder.CreateDiv(LHS, LHSFixedSema, RHS, RHSFixedSema,
+                            CGF.CGM.getCodeGenOpts().OutlineFixedPointMulDiv);
     break;
   case BO_ShlAssign:
   case BO_Shl:

clang/lib/Driver/ToolChains/Clang.cpp

@@ -6369,6 +6369,9 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   Args.AddLastArg(CmdArgs, options::OPT_fexperimental_omit_vtable_rtti,
                   options::OPT_fno_experimental_omit_vtable_rtti);
 
+  Args.AddLastArg(CmdArgs, options::OPT_foutline_fixed_point_mul_div_intrinsics,
+                  options::OPT_fno_outline_fixed_point_mul_div_intrinsics);
+
   if (Arg *A = Args.getLastArg(options::OPT_ffuchsia_api_level_EQ))
     A->render(Args, CmdArgs);
 

llvm/include/llvm/IR/FixedPointBuilder.h

@@ -132,6 +132,44 @@ template <class IRBuilderTy> class FixedPointBuilder {
     return Type::getFloatingPointTy(Ty->getContext(), *FloatSema);
   }
 
+  static SmallString<16> GetOutlinedFuncName(StringRef OpName, bool Saturated,
+                                             unsigned Scale) {
+    SmallString<16> OutlinedFuncName("__outlined_");
+    OutlinedFuncName += OpName;
+    OutlinedFuncName += "_fix";
+    if (Saturated)
+      OutlinedFuncName += "_sat";
+    OutlinedFuncName += "_";
+    OutlinedFuncName += std::to_string(Scale);
+    return OutlinedFuncName;
+  }
+
+  Value *CallFixedPointIntrinsicWrapper(Intrinsic::ID IID,
+                                        StringRef OutlinedFuncName,
+                                        Value *WideLHS, Value *WideRHS,
+                                        unsigned Scale) {
+    Module *M = B.GetInsertBlock()->getParent()->getParent();
+    FunctionCallee Callee =
+        M->getOrInsertFunction(OutlinedFuncName, WideLHS->getType(),
+                               WideLHS->getType(), WideRHS->getType());
+    Function *OutlinedFunc = cast<Function>(Callee.getCallee());
+    if (OutlinedFunc->empty()) {
+      BasicBlock *BB =
+          BasicBlock::Create(M->getContext(), "entry", OutlinedFunc);
+      IRBuilder<> Builder(BB);
+      Value *V = Builder.CreateIntrinsic(IID, {WideLHS->getType()},
+                                         {OutlinedFunc->getArg(0),
+                                          OutlinedFunc->getArg(1),
+                                          Builder.getInt32(Scale)});
+      Builder.CreateRet(V);
+
+      Comdat *C = M->getOrInsertComdat(OutlinedFuncName);
+      OutlinedFunc->setComdat(C);
+      OutlinedFunc->addFnAttr(Attribute::NoInline);
+    }
+    return B.CreateCall(Callee, {WideLHS, WideRHS});
+  }
+
 public:
   FixedPointBuilder(IRBuilderTy &Builder) : B(Builder) {}
 
@@ -285,8 +323,8 @@ template <class IRBuilderTy> class FixedPointBuilder {
   /// \p LHSSema - The semantic of the left hand side
   /// \p RHS     - The right hand side
   /// \p RHSSema - The semantic of the right hand side
-  Value *CreateMul(Value *LHS, const FixedPointSemantics &LHSSema,
-                   Value *RHS, const FixedPointSemantics &RHSSema) {
+  Value *CreateMul(Value *LHS, const FixedPointSemantics &LHSSema, Value *RHS,
+                   const FixedPointSemantics &RHSSema, bool Outlined = false) {
     auto CommonSema = getCommonBinopSemantic(LHSSema, RHSSema);
     bool UseSigned = CommonSema.isSigned() || CommonSema.hasUnsignedPadding();
 
@@ -299,9 +337,19 @@ template <class IRBuilderTy> class FixedPointBuilder {
     } else {
       IID = UseSigned ? Intrinsic::smul_fix : Intrinsic::umul_fix;
     }
-    Value *Result = B.CreateIntrinsic(
-        IID, {WideLHS->getType()},
-        {WideLHS, WideRHS, B.getInt32(CommonSema.getScale())});
+
+    Value *Result;
+    if (!Outlined) {
+      Result = B.CreateIntrinsic(
+          IID, {WideLHS->getType()},
+          {WideLHS, WideRHS, B.getInt32(CommonSema.getScale())});
+    } else {
+      auto OutlinedFuncName =
+          GetOutlinedFuncName(UseSigned ? "smul" : "umul",
+                              CommonSema.isSaturated(), CommonSema.getScale());
+      Result = CallFixedPointIntrinsicWrapper(IID, OutlinedFuncName, WideLHS,
+                                              WideRHS, CommonSema.getScale());
+    }
 
     return CreateFixedToFixed(Result, CommonSema,
                               LHSSema.getCommonSemantics(RHSSema));
@@ -313,8 +361,8 @@ template <class IRBuilderTy> class FixedPointBuilder {
   /// \p LHSSema - The semantic of the left hand side
   /// \p RHS     - The right hand side
   /// \p RHSSema - The semantic of the right hand side
-  Value *CreateDiv(Value *LHS, const FixedPointSemantics &LHSSema,
-                   Value *RHS, const FixedPointSemantics &RHSSema) {
+  Value *CreateDiv(Value *LHS, const FixedPointSemantics &LHSSema, Value *RHS,
+                   const FixedPointSemantics &RHSSema, bool Outlined = false) {
     auto CommonSema = getCommonBinopSemantic(LHSSema, RHSSema);
     bool UseSigned = CommonSema.isSigned() || CommonSema.hasUnsignedPadding();
 
@@ -327,9 +375,19 @@ template <class IRBuilderTy> class FixedPointBuilder {
     } else {
       IID = UseSigned ? Intrinsic::sdiv_fix : Intrinsic::udiv_fix;
     }
-    Value *Result = B.CreateIntrinsic(
-        IID, {WideLHS->getType()},
-        {WideLHS, WideRHS, B.getInt32(CommonSema.getScale())});
+
+    Value *Result;
+    if (!Outlined) {
+      Result = B.CreateIntrinsic(
+          IID, {WideLHS->getType()},
+          {WideLHS, WideRHS, B.getInt32(CommonSema.getScale())});
+    } else {
+      auto OutlinedFuncName =
+          GetOutlinedFuncName(UseSigned ? "sdiv" : "udiv",
+                              CommonSema.isSaturated(), CommonSema.getScale());
+      Result = CallFixedPointIntrinsicWrapper(IID, OutlinedFuncName, WideLHS,
+                                              WideRHS, CommonSema.getScale());
+    }
 
     return CreateFixedToFixed(Result, CommonSema,
                               LHSSema.getCommonSemantics(RHSSema));