[clang] Lower non-builtin sincos[f|l] calls to llvm.sincos.* when -fno-math-errno is set

clang/lib/CodeGen/CGBuiltin.cpp

@@ -2937,12 +2937,21 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
       &getTarget().getLongDoubleFormat() == &llvm::APFloat::IEEEquad())
     BuiltinID = mutateLongDoubleBuiltin(BuiltinID);
 
-  // If the builtin has been declared explicitly with an assembler label,
-  // disable the specialized emitting below. Ideally we should communicate the
-  // rename in IR, or at least avoid generating the intrinsic calls that are
-  // likely to get lowered to the renamed library functions.
-  const unsigned BuiltinIDIfNoAsmLabel =
-      FD->hasAttr<AsmLabelAttr>() ? 0 : BuiltinID;
+  const unsigned BuiltinIDIfEmitIntrinsics = [&] {
+    // If the builtin has been declared explicitly with an assembler label,
+    // disable the specialized emitting below. Ideally we should communicate the
+    // rename in IR, or at least avoid generating the intrinsic calls that are
+    // likely to get lowered to the renamed library functions.
+    if (FD->hasAttr<AsmLabelAttr>())
+      return 0U;
+
+    // If the target requests not to use intrinsics for a builtin, disable the
+    // specialized emitting below.
+    if (!getTargetHooks().shouldUseIntrinsicsForBuiltin(BuiltinID))
+      return 0U;
+
+    return BuiltinID;
+  }();
 
   std::optional<bool> ErrnoOverriden;
   // ErrnoOverriden is true if math-errno is overriden via the
@@ -3035,7 +3044,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
           ConstWithoutErrnoOrExceptions && ErrnoOverridenToFalseWithOpt;
   }
   if (GenerateIntrinsics) {
-    switch (BuiltinIDIfNoAsmLabel) {
+    switch (BuiltinIDIfEmitIntrinsics) {
     case Builtin::BIacos:
     case Builtin::BIacosf:
     case Builtin::BIacosl:
@@ -3377,6 +3386,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
       return RValue::get(emitUnaryMaybeConstrainedFPBuiltin(
           *this, E, Intrinsic::sinh, Intrinsic::experimental_constrained_sinh));
 
+    case Builtin::BIsincos:
+    case Builtin::BIsincosf:
+    case Builtin::BIsincosl:
     case Builtin::BI__builtin_sincos:
     case Builtin::BI__builtin_sincosf:
     case Builtin::BI__builtin_sincosf16:
@@ -3512,7 +3524,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     }
   };
 
-  switch (BuiltinIDIfNoAsmLabel) {
+  switch (BuiltinIDIfEmitIntrinsics) {
   default: break;
   case Builtin::BI__builtin___CFStringMakeConstantString:
   case Builtin::BI__builtin___NSStringMakeConstantString:
@@ -3642,7 +3654,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
   case Builtin::BI__builtin_ctzl:
   case Builtin::BI__builtin_ctzll:
   case Builtin::BI__builtin_ctzg: {
-    bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_ctzg &&
+    bool HasFallback = BuiltinIDIfEmitIntrinsics == Builtin::BI__builtin_ctzg &&
                        E->getNumArgs() > 1;
 
     Value *ArgValue =
@@ -3674,7 +3686,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
   case Builtin::BI__builtin_clzl:
   case Builtin::BI__builtin_clzll:
   case Builtin::BI__builtin_clzg: {
-    bool HasFallback = BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_clzg &&
+    bool HasFallback = BuiltinIDIfEmitIntrinsics == Builtin::BI__builtin_clzg &&
                        E->getNumArgs() > 1;
 
     Value *ArgValue =
@@ -4344,7 +4356,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
       Ty = VecTy->getElementType();
     bool IsSigned = Ty->isSignedIntegerType();
     unsigned Opc;
-    if (BuiltinIDIfNoAsmLabel == Builtin::BI__builtin_elementwise_add_sat)
+    if (BuiltinIDIfEmitIntrinsics == Builtin::BI__builtin_elementwise_add_sat)
       Opc = IsSigned ? llvm::Intrinsic::sadd_sat : llvm::Intrinsic::uadd_sat;
     else
       Opc = IsSigned ? llvm::Intrinsic::ssub_sat : llvm::Intrinsic::usub_sat;

clang/lib/CodeGen/TargetInfo.h

@@ -454,6 +454,23 @@ class TargetCodeGenInfo {
   initBranchProtectionFnAttributes(const TargetInfo::BranchProtectionInfo &BPI,
                                    llvm::AttrBuilder &FuncAttrs);
 
+  /// Returns true if an intrinsic should be emitted for a specific builtin. By
+  /// default, this disables emitting intrinsics for (non-prefixed) sincos
+  /// builtins, as on targets that do not set llvm::TargetSubtargetInfo::useAA()
+  /// the intrinsics can worsen codegen.
+  /// TODO: Remove once the intrinsic lowering works well for all targets.
+  virtual bool shouldUseIntrinsicsForBuiltin(unsigned BuiltinID) const {
+    switch (BuiltinID) {
+    case Builtin::BIsincos:
+    case Builtin::BIsincosf:
+    case Builtin::BIsincosl:
+      return false;
+    default:
+      break;
+    }
+    return true;
+  }
+
 protected:
   static std::string qualifyWindowsLibrary(StringRef Lib);
 

clang/lib/CodeGen/Targets/AArch64.cpp

@@ -181,6 +181,10 @@ class AArch64TargetCodeGenInfo : public TargetCodeGenInfo {
   bool wouldInliningViolateFunctionCallABI(
       const FunctionDecl *Caller, const FunctionDecl *Callee) const override;
 
+  bool shouldUseIntrinsicsForBuiltin(unsigned BuiltinID) const override {
+    return true;
+  }
+
 private:
   // Diagnose calls between functions with incompatible Streaming SVE
   // attributes.

clang/test/CodeGen/AArch64/sincos.c

@@ -1,6 +1,10 @@
 // RUN: %clang_cc1 -triple=aarch64-gnu-linux -emit-llvm -O1 %s -o - | FileCheck --check-prefix=NO-MATH-ERRNO %s
 // RUN: %clang_cc1 -triple=aarch64-gnu-linux -emit-llvm -fmath-errno %s -o - | FileCheck --check-prefix=MATH-ERRNO %s
 
+void sincos(double, double*, double*);
+void sincosf(float, float*, float*);
+void sincosl(long double, long double*, long double*);
+
 // NO-MATH-ERRNO-LABEL: @sincos_f32
 //      NO-MATH-ERRNO:    [[SINCOS:%.*]] = tail call { float, float } @llvm.sincos.f32(float {{.*}})
 // NO-MATH-ERRNO-NEXT:    [[SIN:%.*]] = extractvalue { float, float } [[SINCOS]], 0
@@ -12,6 +16,20 @@
 //      MATH-ERRNO:    call void @sincosf(
 //
 void sincos_f32(float x, float* fp0, float* fp1) {
+  sincosf(x, fp0, fp1);
+}
+
+// NO-MATH-ERRNO-LABEL: @sincos_builtin_f32
+//      NO-MATH-ERRNO:    [[SINCOS:%.*]] = tail call { float, float } @llvm.sincos.f32(float {{.*}})
+// NO-MATH-ERRNO-NEXT:    [[SIN:%.*]] = extractvalue { float, float } [[SINCOS]], 0
+// NO-MATH-ERRNO-NEXT:    [[COS:%.*]] = extractvalue { float, float } [[SINCOS]], 1
+// NO-MATH-ERRNO-NEXT:    store float [[SIN]], ptr {{.*}}, align 4, !alias.scope [[SINCOS_ALIAS_SCOPE:![0-9]+]]
+// NO-MATH-ERRNO-NEXT:    store float [[COS]], ptr {{.*}}, align 4, !noalias [[SINCOS_ALIAS_SCOPE]]
+//
+// MATH-ERRNO-LABEL: @sincos_builtin_f32
+//      MATH-ERRNO:    call void @sincosf(
+//
+void sincos_builtin_f32(float x, float* fp0, float* fp1) {
   __builtin_sincosf(x, fp0, fp1);
 }
 
@@ -26,6 +44,20 @@ void sincos_f32(float x, float* fp0, float* fp1) {
 //      MATH-ERRNO:    call void @sincos(
 //
 void sincos_f64(double x, double* dp0, double* dp1) {
+  sincos(x, dp0, dp1);
+}
+
+// NO-MATH-ERRNO-LABEL: @sincos_builtin_f64
+//      NO-MATH-ERRNO:    [[SINCOS:%.*]] = tail call { double, double } @llvm.sincos.f64(double {{.*}})
+// NO-MATH-ERRNO-NEXT:    [[SIN:%.*]] = extractvalue { double, double } [[SINCOS]], 0
+// NO-MATH-ERRNO-NEXT:    [[COS:%.*]] = extractvalue { double, double } [[SINCOS]], 1
+// NO-MATH-ERRNO-NEXT:    store double [[SIN]], ptr {{.*}}, align 8, !alias.scope [[SINCOS_ALIAS_SCOPE:![0-9]+]]
+// NO-MATH-ERRNO-NEXT:    store double [[COS]], ptr {{.*}}, align 8, !noalias [[SINCOS_ALIAS_SCOPE]]
+//
+// MATH-ERRNO-LABEL: @sincos_builtin_f64
+//      MATH-ERRNO:    call void @sincos(
+//
+void sincos_builtin_f64(double x, double* dp0, double* dp1) {
   __builtin_sincos(x, dp0, dp1);
 }
 
@@ -40,5 +72,19 @@ void sincos_f64(double x, double* dp0, double* dp1) {
 //      MATH-ERRNO:    call void @sincosl(
 //
 void sincos_f128(long double x, long double* ldp0, long double* ldp1) {
+  sincosl(x, ldp0, ldp1);
+}
+
+// NO-MATH-ERRNO-LABEL: @sincos_builtin_f128
+//      NO-MATH-ERRNO:    [[SINCOS:%.*]] = tail call { fp128, fp128 } @llvm.sincos.f128(fp128 {{.*}})
+// NO-MATH-ERRNO-NEXT:    [[SIN:%.*]] = extractvalue { fp128, fp128 } [[SINCOS]], 0
+// NO-MATH-ERRNO-NEXT:    [[COS:%.*]] = extractvalue { fp128, fp128 } [[SINCOS]], 1
+// NO-MATH-ERRNO-NEXT:    store fp128 [[SIN]], ptr {{.*}}, align 16, !alias.scope [[SINCOS_ALIAS_SCOPE:![0-9]+]]
+// NO-MATH-ERRNO-NEXT:    store fp128 [[COS]], ptr {{.*}}, align 16, !noalias [[SINCOS_ALIAS_SCOPE]]
+//
+// MATH-ERRNO-LABEL: @sincos_builtin_f128
+//      MATH-ERRNO:    call void @sincosl(
+//
+void sincos_builtin_f128(long double x, long double* ldp0, long double* ldp1) {
   __builtin_sincosl(x, ldp0, ldp1);
 }

clang/test/CodeGen/math-libcalls.c

@@ -660,6 +660,17 @@ void foo(double *d, float f, float *fp, long double *l, int *i, const char *c) {
 // HAS_MAYTRAP: declare float @llvm.experimental.constrained.sinh.f32(
 // HAS_MAYTRAP: declare x86_fp80 @llvm.experimental.constrained.sinh.f80(
 
+sincos(f, d, d);       sincosf(f, fp, fp);        sincosl(f, l, l);
+
+// NO__ERRNO: declare void @sincos(double noundef, ptr noundef, ptr noundef) [[NOT_READNONE]]
+// NO__ERRNO: declare void @sincosf(float noundef, ptr noundef, ptr noundef) [[NOT_READNONE]]
+// NO__ERRNO: declare void @sincosl(x86_fp80 noundef, ptr noundef, ptr noundef) [[NOT_READNONE]]
+// HAS__ERRNO: declare void @sincos(double noundef, ptr noundef, ptr noundef) [[NOT_READNONE]]
+// HAS__ERRNO: declare void @sincosf(float noundef, ptr noundef, ptr noundef) [[NOT_READNONE]]
+// HAS__ERRNO: declare void @sincosl(x86_fp80 noundef, ptr noundef, ptr noundef) [[NOT_READNONE]]
+// HAS_MAYTRAP: declare void @sincos(double noundef, ptr noundef, ptr noundef) [[NOT_READNONE]]
+// HAS_MAYTRAP: declare void @sincosf(float noundef, ptr noundef, ptr noundef) [[NOT_READNONE]]
+// HAS_MAYTRAP: declare void @sincosl(x86_fp80 noundef, ptr noundef, ptr noundef) [[NOT_READNONE]]
 
   sqrt(f);       sqrtf(f);      sqrtl(f);