[Inliner] Don't count a call penalty for foldable __memcpy_chk and similar

When the size is an appropriate constant, __memcpy_chk will turn into a
memcpy that gets folded away by InstCombine. Therefore this patch avoids
counting these as calls for purposes of inlining costs.

This is only really relevant on platforms whose headers redirect memcpy to
__memcpy_chk (such as Mac). On platforms that use intrinsics, memcpy and
similar functions are already exempt from call penalties.

Author: citymarina

llvm/lib/Analysis/InlineCost.cpp

@@ -436,6 +436,7 @@ class CallAnalyzer : public InstVisitor<CallAnalyzer, bool> {
   bool simplifyIntrinsicCallIsConstant(CallBase &CB);
   bool simplifyIntrinsicCallObjectSize(CallBase &CB);
   ConstantInt *stripAndComputeInBoundsConstantOffsets(Value *&V);
+  bool isLoweredToCall(Function *F, CallBase &Call);
 
   /// Return true if the given argument to the function being considered for
   /// inlining has the given attribute set either at the call site or the
@@ -2270,6 +2271,48 @@ bool CallAnalyzer::simplifyCallSite(Function *F, CallBase &Call) {
   return false;
 }
 
+bool CallAnalyzer::isLoweredToCall(Function *F, CallBase &Call) {
+  // Calls to memcpy with a known constant size 1/2/4/8 should not incur a call
+  // penalty, as the calls will be folded away by InstCombine. This is only
+  // really relevant on platforms whose headers redirect memcpy to __memcpy_chk
+  // (e.g. Mac), as other platforms use memcpy intrinsics, which are already
+  // exempt from the call penalty.
+  if (GetTLI) {
+    auto TLI = GetTLI(*F);
+    LibFunc LF;
+    if (TLI.getLibFunc(*F, LF) && TLI.has(LF)) {
+      switch (LF) {
+      case LibFunc_memcpy_chk:
+      case LibFunc_memmove_chk:
+      case LibFunc_mempcpy_chk:
+      case LibFunc_memset_chk: {
+        auto LenOp = dyn_cast_or_null<ConstantInt>(Call.getOperand(2));
+        if (!LenOp)
+          LenOp = dyn_cast_or_null<ConstantInt>(
+              SimplifiedValues.lookup(Call.getOperand(2)));
+        auto ObjSizeOp = dyn_cast_or_null<ConstantInt>(Call.getOperand(3));
+        if (!ObjSizeOp)
+          ObjSizeOp = dyn_cast_or_null<ConstantInt>(
+              SimplifiedValues.lookup(Call.getOperand(3)));
+        if (LenOp && ObjSizeOp) {
+          auto Len = LenOp->getLimitedValue();
+          auto ObjSize = ObjSizeOp->getLimitedValue();
+          if (ObjSize >= Len &&
+              (Len == 1 || Len == 2 || Len == 4 || Len == 8)) {
+            return false;
+          }
+        }
+        break;
+      }
+      default:
+        break;
+      }
+    }
+  }
+
+  return TTI.isLoweredToCall(F);
+}
+
 bool CallAnalyzer::visitCallBase(CallBase &Call) {
   if (!onCallBaseVisitStart(Call))
     return true;
@@ -2351,7 +2394,7 @@ bool CallAnalyzer::visitCallBase(CallBase &Call) {
       return false;
   }
 
-  if (TTI.isLoweredToCall(F)) {
+  if (isLoweredToCall(F, Call)) {
     onLoweredCall(F, Call, IsIndirectCall);
   }
 

llvm/test/Transforms/Inline/AArch64/memcpy-constant-size.ll

@@ -38,7 +38,11 @@ define void @caller3(ptr %dst, ptr %src) {
 define void @caller4(ptr %dst, ptr %src) {
 ; CHECK-LABEL: define void @caller4
 ; CHECK-SAME: (ptr [[DST:%.*]], ptr [[SRC:%.*]]) {
-; CHECK-NEXT:    call void @callee(ptr [[DST]], ptr [[SRC]], i64 4)
+; CHECK-NEXT:    [[OBJSIZE_I:%.*]] = call i64 @llvm.objectsize.i64.p0(ptr [[DST]], i1 false, i1 true, i1 false)
+; CHECK-NEXT:    [[CALL_MEMCPY_I:%.*]] = call ptr @__memcpy_chk(ptr [[DST]], ptr [[SRC]], i64 4, i64 [[OBJSIZE_I]])
+; CHECK-NEXT:    [[CALL_MEMMOVE_I:%.*]] = call ptr @__memmove_chk(ptr [[DST]], ptr [[SRC]], i64 4, i64 [[OBJSIZE_I]])
+; CHECK-NEXT:    [[CALL_MEMPCPY_I:%.*]] = call ptr @__mempcpy_chk(ptr [[DST]], ptr [[SRC]], i64 4, i64 [[OBJSIZE_I]])
+; CHECK-NEXT:    [[CALL_MEMSET_I:%.*]] = call ptr @__memset_chk(ptr [[DST]], i32 0, i64 4, i64 [[OBJSIZE_I]])
 ; CHECK-NEXT:    ret void
 ;
   call void @callee(ptr %dst, ptr %src, i64 4)

llvm/test/Transforms/PhaseOrdering/AArch64/memcpy-constant-size.ll

@@ -54,7 +54,8 @@ define void @callee_memset(ptr %dst, i64 %size) {
 define void @caller_memcpy(ptr %dst, ptr %src) {
 ; CHECK-LABEL: define void @caller_memcpy
 ; CHECK-SAME: (ptr [[DST:%.*]], ptr nocapture readonly [[SRC:%.*]]) local_unnamed_addr #[[ATTR0]] {
-; CHECK-NEXT:    tail call void @callee_memcpy(ptr [[DST]], ptr [[SRC]], i64 4)
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[SRC]], align 1
+; CHECK-NEXT:    store i32 [[TMP1]], ptr [[DST]], align 1
 ; CHECK-NEXT:    ret void
 ;
   call void @callee_memcpy(ptr %dst, ptr %src, i64 4)
@@ -64,7 +65,8 @@ define void @caller_memcpy(ptr %dst, ptr %src) {
 define void @caller_memmove(ptr %dst, ptr %src) {
 ; CHECK-LABEL: define void @caller_memmove
 ; CHECK-SAME: (ptr [[DST:%.*]], ptr [[SRC:%.*]]) local_unnamed_addr #[[ATTR1]] {
-; CHECK-NEXT:    tail call void @callee_memmove(ptr [[DST]], ptr [[SRC]], i64 4)
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[SRC]], align 1
+; CHECK-NEXT:    store i32 [[TMP1]], ptr [[DST]], align 1
 ; CHECK-NEXT:    ret void
 ;
   call void @callee_memmove(ptr %dst, ptr %src, i64 4)
@@ -74,7 +76,8 @@ define void @caller_memmove(ptr %dst, ptr %src) {
 define void @caller_mempcpy(ptr %dst, ptr %src) {
 ; CHECK-LABEL: define void @caller_mempcpy
 ; CHECK-SAME: (ptr [[DST:%.*]], ptr [[SRC:%.*]]) local_unnamed_addr #[[ATTR1]] {
-; CHECK-NEXT:    tail call void @callee_mempcpy(ptr [[DST]], ptr [[SRC]], i64 4)
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[SRC]], align 1
+; CHECK-NEXT:    store i32 [[TMP1]], ptr [[DST]], align 1
 ; CHECK-NEXT:    ret void
 ;
   call void @callee_mempcpy(ptr %dst, ptr %src, i64 4)
@@ -84,7 +87,7 @@ define void @caller_mempcpy(ptr %dst, ptr %src) {
 define void @caller_memset(ptr %dst) {
 ; CHECK-LABEL: define void @caller_memset
 ; CHECK-SAME: (ptr [[DST:%.*]]) local_unnamed_addr #[[ATTR0]] {
-; CHECK-NEXT:    tail call void @callee_memset(ptr [[DST]], i64 4)
+; CHECK-NEXT:    store i32 0, ptr [[DST]], align 1
 ; CHECK-NEXT:    ret void
 ;
   call void @callee_memset(ptr %dst, i64 4)