[licm] clone metadata when hoisting conditional branch

Author: mtrofin

llvm/include/llvm/Transforms/Scalar/LICM.h

@@ -44,6 +44,7 @@ class LoopNest;
 
 extern cl::opt<unsigned> SetLicmMssaOptCap;
 extern cl::opt<unsigned> SetLicmMssaNoAccForPromotionCap;
+LLVM_ABI extern cl::opt<bool> ProfcheckDisableMetadataFixes;
 
 struct LICMOptions {
   unsigned MssaOptCap;

llvm/lib/IR/Instruction.cpp

@@ -26,9 +26,20 @@
 #include "llvm/IR/Operator.h"
 #include "llvm/IR/ProfDataUtils.h"
 #include "llvm/IR/Type.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Compiler.h"
 using namespace llvm;
 
+namespace llvm {
+// FIXME: Flag used for an ablation performance test, Issue #147390. Placing it
+// here because referencing IR should be feasible from anywhere. Will be
+// removed after the ablation test.
+LLVM_ABI cl::opt<bool> ProfcheckDisableMetadataFixes(
+    "profcheck-disable-metadata-fixes", cl::Hidden, cl::init(false),
+    cl::desc(
+        "Disable metadata propagation fixes discovered through Issue #147390"));
+} // namespace llvm
+
 InsertPosition::InsertPosition(Instruction *InsertBefore)
     : InsertAt(InsertBefore ? InsertBefore->getIterator()
                             : InstListType::iterator()) {}

llvm/lib/Transforms/Scalar/LICM.cpp

@@ -857,9 +857,19 @@ class ControlFlowHoister {
     }
 
     // Now finally clone BI.
-    ReplaceInstWithInst(
-        HoistTarget->getTerminator(),
-        BranchInst::Create(HoistTrueDest, HoistFalseDest, BI->getCondition()));
+    auto *NewBI =
+        BranchInst::Create(HoistTrueDest, HoistFalseDest, BI->getCondition(),
+                           HoistTarget->getTerminator()->getIterator());
+    HoistTarget->getTerminator()->eraseFromParent();
+    // Handle "performance-related" metadata. In particular:
+    // - md_prof: it should also come from the original branch - since the
+    // condition was hoisted, the branch probabilities shouldn't change.
+    if (!ProfcheckDisableMetadataFixes)
+      NewBI->copyMetadata(*BI, {LLVMContext::MD_prof});
+    // FIXME: Issue #152767: debug info should also be the same as the
+    // original branch, **if** the user explicitly indicated that.
+    NewBI->setDebugLoc(HoistTarget->getTerminator()->getDebugLoc());
+
     ++NumClonedBranches;
 
     assert(CurLoop->getLoopPreheader() &&

llvm/test/Transforms/LICM/hoist-phi-metadata.ll

@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals --version 2
+; Test that hoisting conditional branches copies the debug and profiling info
+; metadata from the branch being hoisted.
+; RUN: opt -S -passes=licm -licm-control-flow-hoisting=1 %s -o - | FileCheck %s
+
+define void @triangle_phi(i32 %x, ptr %p) {
+; CHECK-LABEL: define void @triangle_phi
+; CHECK-SAME: (i32 [[X:%.*]], ptr [[P:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp sgt i32 [[X]], 0
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF_LICM:%.*]], label [[THEN_LICM:%.*]], !prof [[PROF2:![0-9]+]]
+; CHECK:       if.licm:
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X]], 1
+; CHECK-NEXT:    br label [[THEN_LICM]]
+; CHECK:       then.licm:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i32 [ [[ADD]], [[IF_LICM]] ], [ [[X]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    store i32 [[PHI]], ptr [[P]], align 4
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ne i32 [[PHI]], 0
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    br i1 [[CMP1]], label [[IF:%.*]], label [[THEN:%.*]], !dbg [[DBG3:![0-9]+]], !prof [[PROF2]]
+; CHECK:       if:
+; CHECK-NEXT:    br label [[THEN]]
+; CHECK:       then:
+; CHECK-NEXT:    br i1 [[CMP2]], label [[LOOP]], label [[END:%.*]], !dbg [[DBG7:![0-9]+]], !prof [[PROF8:![0-9]+]]
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop, !dbg !5
+loop:
+  %cmp1 = icmp sgt i32 %x, 0
+  br i1 %cmp1, label %if, label %then, !dbg !6, !prof !8
+if:
+  %add = add i32 %x, 1
+  br label %then
+
+then:
+  %phi = phi i32 [ %add, %if ], [ %x, %loop ]
+  store i32 %phi, ptr %p
+  %cmp2 = icmp ne i32 %phi, 0
+  br i1 %cmp2, label %loop, label %end, !dbg !7, !prof !9
+
+end:
+  ret void
+}
+
+!llvm.module.flags = !{!2, !3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1)
+!1 = !DIFile(filename: "t", directory: "/")
+!2 = !{i32 7, !"Dwarf Version", i32 5}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = distinct !DISubprogram(name: "triangle_phi", linkageName: "triangle_phi", scope: !1, file: !1, line: 1, unit: !0)
+!5 = !DILocation(line: 1, column: 22, scope: !4)
+!6 = !DILocation(line: 2, column: 22, scope: !4)
+!7 = !DILocation(line: 3, column: 22, scope: !4)
+!8 = !{!"branch_weights", i32 5, i32 7}
+!9 = !{!"branch_weights", i32 13, i32 11}
+;.
+; CHECK: [[META0:![0-9]+]] = !{i32 7, !"Dwarf Version", i32 5}
+; CHECK: [[META1:![0-9]+]] = !{i32 2, !"Debug Info Version", i32 3}
+; CHECK: [[PROF2]] = !{!"branch_weights", i32 5, i32 7}
+; CHECK: [[DBG3]] = !DILocation(line: 2, column: 22, scope: [[META4:![0-9]+]])
+; CHECK: [[META4]] = distinct !DISubprogram(name: "triangle_phi", linkageName: "triangle_phi", scope: [[META5:![0-9]+]], file: [[META5]], line: 1, spFlags: DISPFlagDefinition, unit: [[META6:![0-9]+]])
+; CHECK: [[META5]] = !DIFile(filename: "{{.*}}t", directory: {{.*}})
+; CHECK: [[META6]] = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: [[META5]], isOptimized: false, runtimeVersion: 0, emissionKind: NoDebug)
+; CHECK: [[DBG7]] = !DILocation(line: 3, column: 22, scope: [[META4]])
+; CHECK: [[PROF8]] = !{!"branch_weights", i32 13, i32 11}
+;.