Use the Propeller CFG profile in the PGO analysis map if it is available.

llvm/include/llvm/CodeGen/BasicBlockSectionsProfileReader.h

@@ -42,18 +42,40 @@ struct BBClusterInfo {
   unsigned PositionInCluster;
 };
 
+// This represents the CFG profile data for a function.
+struct CfgProfile {
+  // Node counts for each basic block.
+  DenseMap<UniqueBBID, uint64_t> NodeCounts;
+  // Edge counts for each edge, stored as a nested map.
+  DenseMap<UniqueBBID, DenseMap<UniqueBBID, uint64_t>> EdgeCounts;
+
+  // Returns the profile count for the given basic block or zero if it does not
+  // exist.
+  uint64_t getBlockCount(const UniqueBBID &BBID) const {
+    return NodeCounts.lookup(BBID);
+  }
+
+  // Returns the profile count for the edge from `SrcBBID` to `SinkBBID` or
+  // zero if it does not exist.
+  uint64_t getEdgeCount(const UniqueBBID &SrcBBID,
+                        const UniqueBBID &SinkBBID) const {
+    auto It = EdgeCounts.find(SrcBBID);
+    if (It == EdgeCounts.end())
+      return 0;
+    return It->second.lookup(SinkBBID);
+  }
+};
+
 // This represents the raw input profile for one function.
-struct FunctionPathAndClusterInfo {
+struct FunctionProfile {
   // BB Cluster information specified by `UniqueBBID`s.
   SmallVector<BBClusterInfo> ClusterInfo;
   // Paths to clone. A path a -> b -> c -> d implies cloning b, c, and d along
   // the edge a -> b (a is not cloned). The index of the path in this vector
   // determines the `UniqueBBID::CloneID` of the cloned blocks in that path.
   SmallVector<SmallVector<unsigned>> ClonePaths;
-  // Node counts for each basic block.
-  DenseMap<UniqueBBID, uint64_t> NodeCounts;
-  // Edge counts for each edge, stored as a nested map.
-  DenseMap<UniqueBBID, DenseMap<UniqueBBID, uint64_t>> EdgeCounts;
+  // Cfg profile data (block and edge frequencies).
+  CfgProfile Cfg;
   // Hash for each basic block. The Hashes are stored for every original block
   // (not cloned blocks), hence the map key being unsigned instead of
   // UniqueBBID.
@@ -85,10 +107,14 @@ class BasicBlockSectionsProfileReader {
   SmallVector<SmallVector<unsigned>>
   getClonePathsForFunction(StringRef FuncName) const;
 
-  // Returns the profile count for the edge from `SrcBBID` to `SinkBBID` in
-  // function `FuncName` or zero if it does not exist.
-  uint64_t getEdgeCount(StringRef FuncName, const UniqueBBID &SrcBBID,
-                        const UniqueBBID &SinkBBID) const;
+  // Returns a pointer to the CfgProfile for the given function.
+  // Returns nullptr if no profile data is available for the function.
+  const CfgProfile *getFunctionCfgProfile(StringRef FuncName) const {
+    auto It = ProgramPathAndClusterInfo.find(getAliasName(FuncName));
+    if (It == ProgramPathAndClusterInfo.end())
+      return nullptr;
+    return &It->second.Cfg;
+  }
 
 private:
   StringRef getAliasName(StringRef FuncName) const {
@@ -136,7 +162,7 @@ class BasicBlockSectionsProfileReader {
   // for (all or some of) its basic blocks. The cluster information for every
   // basic block includes its cluster ID along with the position of the basic
   // block in that cluster.
-  StringMap<FunctionPathAndClusterInfo> ProgramPathAndClusterInfo;
+  StringMap<FunctionProfile> ProgramPathAndClusterInfo;
 
   // Some functions have alias names. We use this map to find the main alias
   // name which appears in ProgramPathAndClusterInfo as a key.
@@ -196,9 +222,6 @@ class BasicBlockSectionsProfileReaderWrapperPass : public ImmutablePass {
   SmallVector<SmallVector<unsigned>>
   getClonePathsForFunction(StringRef FuncName) const;
 
-  uint64_t getEdgeCount(StringRef FuncName, const UniqueBBID &SrcBBID,
-                        const UniqueBBID &DestBBID) const;
-
   // Initializes the FunctionNameToDIFilename map for the current module and
   // then reads the profile for the matching functions.
   bool doInitialization(Module &M) override;

llvm/include/llvm/MC/MCContext.h

@@ -175,7 +175,7 @@ class MCContext {
   unsigned GetInstance(unsigned LocalLabelVal);
 
   /// SHT_LLVM_BB_ADDR_MAP version to emit.
-  uint8_t BBAddrMapVersion = 4;
+  uint8_t BBAddrMapVersion = 5;
 
   /// The file name of the log file from the environment variable
   /// AS_SECURE_LOG_FILE.  Which must be set before the .secure_log_unique

llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp

@@ -20,7 +20,6 @@
 #include "WinException.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/APInt.h"
-#include "llvm/ADT/BitmaskEnum.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -37,6 +36,7 @@
 #include "llvm/BinaryFormat/COFF.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/BinaryFormat/ELF.h"
+#include "llvm/CodeGen/BasicBlockSectionsProfileReader.h"
 #include "llvm/CodeGen/GCMetadata.h"
 #include "llvm/CodeGen/GCMetadataPrinter.h"
 #include "llvm/CodeGen/LazyMachineBlockFrequencyInfo.h"
@@ -150,6 +150,7 @@ enum class PGOMapFeaturesEnum {
   FuncEntryCount,
   BBFreq,
   BrProb,
+  PropellerCFG,
   All,
 };
 static cl::bits<PGOMapFeaturesEnum> PgoAnalysisMapFeatures(
@@ -166,6 +167,12 @@ static cl::bits<PGOMapFeaturesEnum> PgoAnalysisMapFeatures(
         "Enable extended information within the SHT_LLVM_BB_ADDR_MAP that is "
         "extracted from PGO related analysis."));
 
+static cl::opt<bool> PgoAnalysisMapEmitBBSectionsCfg(
+    "pgo-analysis-map-emit-bb-sections-cfg",
+    cl::desc("Enable the post-link cfg information from the basic block "
+             "sections profile in the PGO analysis map"),
+    cl::Hidden, cl::init(false));
+
 static cl::opt<bool> BBAddrMapSkipEmitBBEntries(
     "basic-block-address-map-skip-bb-entries",
     cl::desc("Skip emitting basic block entries in the SHT_LLVM_BB_ADDR_MAP "
@@ -479,6 +486,7 @@ void AsmPrinter::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addRequired<MachineBranchProbabilityInfoWrapperPass>();
   if (EmitBBHash)
     AU.addRequired<MachineBlockHashInfo>();
+  AU.addUsedIfAvailable<BasicBlockSectionsProfileReaderWrapperPass>();
 }
 
 bool AsmPrinter::doInitialization(Module &M) {
@@ -1411,7 +1419,7 @@ static uint32_t getBBAddrMapMetadata(const MachineBasicBlock &MBB) {
 
 static llvm::object::BBAddrMap::Features
 getBBAddrMapFeature(const MachineFunction &MF, int NumMBBSectionRanges,
-                    bool HasCalls) {
+                    bool HasCalls, const CfgProfile *FuncCfgProfile) {
   // Ensure that the user has not passed in additional options while also
   // specifying all or none.
   if ((PgoAnalysisMapFeatures.isSet(PGOMapFeaturesEnum::None) ||
@@ -1433,17 +1441,17 @@ getBBAddrMapFeature(const MachineFunction &MF, int NumMBBSectionRanges,
   bool BrProbEnabled =
       AllFeatures ||
       (!NoFeatures && PgoAnalysisMapFeatures.isSet(PGOMapFeaturesEnum::BrProb));
+  bool PostLinkCfgEnabled = FuncCfgProfile && PgoAnalysisMapEmitBBSectionsCfg;
 
   if ((BBFreqEnabled || BrProbEnabled) && BBAddrMapSkipEmitBBEntries) {
     MF.getFunction().getContext().emitError(
-        "BB entries info is required for BBFreq and BrProb "
-        "features");
+        "BB entries info is required for BBFreq and BrProb features");
   }
   return {FuncEntryCountEnabled, BBFreqEnabled, BrProbEnabled,
           MF.hasBBSections() && NumMBBSectionRanges > 1,
           // Use static_cast to avoid breakage of tests on windows.
           static_cast<bool>(BBAddrMapSkipEmitBBEntries), HasCalls,
-          static_cast<bool>(EmitBBHash), false};
+          static_cast<bool>(EmitBBHash), PostLinkCfgEnabled};
 }
 
 void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
@@ -1452,6 +1460,14 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
   assert(BBAddrMapSection && ".llvm_bb_addr_map section is not initialized.");
   bool HasCalls = !CurrentFnCallsiteEndSymbols.empty();
 
+  const BasicBlockSectionsProfileReader *BBSPR = nullptr;
+  if (auto *BBSPRPass =
+          getAnalysisIfAvailable<BasicBlockSectionsProfileReaderWrapperPass>())
+    BBSPR = &BBSPRPass->getBBSPR();
+  const CfgProfile *FuncCfgProfile = nullptr;
+  if (BBSPR)
+    FuncCfgProfile = BBSPR->getFunctionCfgProfile(MF.getFunction().getName());
+
   const MCSymbol *FunctionSymbol = getFunctionBegin();
 
   OutStreamer->pushSection();
@@ -1460,7 +1476,8 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
   uint8_t BBAddrMapVersion = OutStreamer->getContext().getBBAddrMapVersion();
   OutStreamer->emitInt8(BBAddrMapVersion);
   OutStreamer->AddComment("feature");
-  auto Features = getBBAddrMapFeature(MF, MBBSectionRanges.size(), HasCalls);
+  auto Features = getBBAddrMapFeature(MF, MBBSectionRanges.size(), HasCalls,
+                                      FuncCfgProfile);
   OutStreamer->emitInt8(Features.encode());
   // Emit BB Information for each basic block in the function.
   if (Features.MultiBBRange) {
@@ -1565,6 +1582,11 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
           OutStreamer->AddComment("basic block frequency");
           OutStreamer->emitULEB128IntValue(
               MBFI->getBlockFreq(&MBB).getFrequency());
+          if (Features.PostLinkCfg) {
+            OutStreamer->AddComment("basic block frequency (propeller)");
+            OutStreamer->emitULEB128IntValue(
+                FuncCfgProfile->getBlockCount(*MBB.getBBID()));
+          }
         }
         if (Features.BrProb) {
           unsigned SuccCount = MBB.succ_size();
@@ -1576,6 +1598,11 @@ void AsmPrinter::emitBBAddrMapSection(const MachineFunction &MF) {
             OutStreamer->AddComment("successor branch probability");
             OutStreamer->emitULEB128IntValue(
                 MBPI->getEdgeProbability(&MBB, SuccMBB).getNumerator());
+            if (Features.PostLinkCfg) {
+              OutStreamer->AddComment("successor branch frequency (propeller)");
+              OutStreamer->emitULEB128IntValue(FuncCfgProfile->getEdgeCount(
+                  *MBB.getBBID(), *SuccMBB->getBBID()));
+            }
           }
         }
       }

llvm/lib/CodeGen/BasicBlockSectionsProfileReader.cpp

@@ -76,21 +76,6 @@ BasicBlockSectionsProfileReader::getClonePathsForFunction(
   return ProgramPathAndClusterInfo.lookup(getAliasName(FuncName)).ClonePaths;
 }
 
-uint64_t BasicBlockSectionsProfileReader::getEdgeCount(
-    StringRef FuncName, const UniqueBBID &SrcBBID,
-    const UniqueBBID &SinkBBID) const {
-  auto It = ProgramPathAndClusterInfo.find(getAliasName(FuncName));
-  if (It == ProgramPathAndClusterInfo.end())
-    return 0;
-  auto NodeIt = It->second.EdgeCounts.find(SrcBBID);
-  if (NodeIt == It->second.EdgeCounts.end())
-    return 0;
-  auto EdgeIt = NodeIt->second.find(SinkBBID);
-  if (EdgeIt == NodeIt->second.end())
-    return 0;
-  return EdgeIt->second;
-}
-
 // Reads the version 1 basic block sections profile. Profile for each function
 // is encoded as follows:
 //   m <module_name>
@@ -279,10 +264,10 @@ Error BasicBlockSectionsProfileReader::ReadV1Profile() {
                 Twine("unsigned integer expected: '") + CountStr + "'");
           if (i == 0) {
             // The first element represents the source and its total count.
-            FI->second.NodeCounts[SrcBBID = *BBID] = Count;
+            FI->second.Cfg.NodeCounts[SrcBBID = *BBID] = Count;
             continue;
           }
-          FI->second.EdgeCounts[SrcBBID][*BBID] = Count;
+          FI->second.Cfg.EdgeCounts[SrcBBID][*BBID] = Count;
         }
       }
       continue;
@@ -506,12 +491,6 @@ BasicBlockSectionsProfileReaderWrapperPass::getClonePathsForFunction(
   return BBSPR.getClonePathsForFunction(FuncName);
 }
 
-uint64_t BasicBlockSectionsProfileReaderWrapperPass::getEdgeCount(
-    StringRef FuncName, const UniqueBBID &SrcBBID,
-    const UniqueBBID &SinkBBID) const {
-  return BBSPR.getEdgeCount(FuncName, SrcBBID, SinkBBID);
-}
-
 BasicBlockSectionsProfileReader &
 BasicBlockSectionsProfileReaderWrapperPass::getBBSPR() {
   return BBSPR;

llvm/test/CodeGen/X86/basic-block-address-map-empty-function.ll

@@ -19,7 +19,7 @@ entry:
 ; CHECK:	func:
 ; CHECK:	.Lfunc_begin1:
 ; CHECK:		.section	.llvm_bb_addr_map,"o",@llvm_bb_addr_map,.text{{$}}
-; CHECK-NEXT:		.byte 4			# version
+; CHECK-NEXT:		.byte 5			# version
 ; BASIC-NEXT:		.byte 0			# feature
 ; PGO-NEXT:		.byte 3			# feature
 ; CHECK-NEXT:		.quad	.Lfunc_begin1	# function address

llvm/test/CodeGen/X86/basic-block-address-map-function-sections.ll

@@ -10,7 +10,7 @@ define dso_local i32 @_Z3barv() {
 ; CHECK-LABEL:	_Z3barv:
 ; CHECK-NEXT:	[[BAR_BEGIN:.Lfunc_begin[0-9]+]]:
 ; CHECK:		.section .llvm_bb_addr_map,"o",@llvm_bb_addr_map,.text._Z3barv{{$}}
-; CHECK-NEXT:		.byte 4			# version
+; CHECK-NEXT:		.byte 5			# version
 ; CHECK-NEXT:		.byte 0			# feature
 ; CHECK-NEXT:		.quad [[BAR_BEGIN]]	# function address
 
@@ -23,7 +23,7 @@ define dso_local i32 @_Z3foov() {
 ; CHECK-LABEL:	_Z3foov:
 ; CHECK-NEXT:	[[FOO_BEGIN:.Lfunc_begin[0-9]+]]:
 ; CHECK:		.section  .llvm_bb_addr_map,"o",@llvm_bb_addr_map,.text._Z3foov{{$}}
-; CHECK-NEXT:		.byte 4			# version
+; CHECK-NEXT:		.byte 5			# version
 ; CHECK-NEXT:		.byte 32                # feature
 ; CHECK-NEXT:		.quad [[FOO_BEGIN]]	# function address
 
@@ -36,6 +36,6 @@ define linkonce_odr dso_local i32 @_Z4fooTIiET_v() comdat {
 ; CHECK-LABEL:	_Z4fooTIiET_v:
 ; CHECK-NEXT:	[[FOOCOMDAT_BEGIN:.Lfunc_begin[0-9]+]]:
 ; CHECK:		.section .llvm_bb_addr_map,"oG",@llvm_bb_addr_map,.text._Z4fooTIiET_v,_Z4fooTIiET_v,comdat{{$}}
-; CHECK-NEXT:		.byte 4				# version
+; CHECK-NEXT:		.byte 5				# version
 ; CHECK-NEXT:		.byte 0				# feature
 ; CHECK-NEXT:		.quad [[FOOCOMDAT_BEGIN]]	# function address

llvm/test/CodeGen/X86/basic-block-address-map-pgo-features.ll

@@ -69,7 +69,7 @@ declare i32 @__gxx_personality_v0(...)
 ; CHECK-LABEL:	.Lfunc_end0:
 
 ; CHECK: 	.section	.llvm_bb_addr_map,"o",@llvm_bb_addr_map,.text._Z3bazb{{$}}
-; CHECK-NEXT:	.byte	4		# version
+; CHECK-NEXT:	.byte	5		# version
 ; BASIC-NEXT:	.byte	32		# feature
 ; PGO-ALL-NEXT:	.byte	39		# feature
 ; FEC-ONLY-NEXT:.byte	33		# feature

llvm/test/CodeGen/X86/basic-block-address-map-with-basic-block-sections.ll

@@ -47,7 +47,7 @@ declare i32 @__gxx_personality_v0(...)
 ; CHECK-LABEL:	.Lfunc_end0:
 
 ; CHECK:		.section	.llvm_bb_addr_map,"o",@llvm_bb_addr_map,.text.hot._Z3bazb
-; CHECK-NEXT:   .byte   4                       # version
+; CHECK-NEXT:   .byte   5                       # version
 ; CHECK-NEXT:   .byte   40                      # feature
 ; CHECK-NEXT:   .byte   2                       # number of basic block ranges
 ; CHECK-NEXT:	.quad	.Lfunc_begin0           # base address

llvm/test/CodeGen/X86/basic-block-address-map-with-emit-bb-hash.ll

@@ -50,7 +50,7 @@ declare i32 @__gxx_personality_v0(...)
 ; UNIQ:			.section	.llvm_bb_addr_map,"o",@llvm_bb_addr_map,.text._Z3bazb{{$}}
 ;; Verify that with -unique-section-names=false, the unique id of the text section gets assigned to the llvm_bb_addr_map section.
 ; NOUNIQ:		.section	.llvm_bb_addr_map,"o",@llvm_bb_addr_map,.text,unique,1
-; CHECK-NEXT:   .byte   4		# version
+; CHECK-NEXT:   .byte   5		# version
 ; CHECK-NEXT:   .byte   96		# feature
 ; CHECK-NEXT:	.quad	.Lfunc_begin0	# function address
 ; CHECK-NEXT:	.byte	6		# number of basic blocks

llvm/test/CodeGen/X86/basic-block-address-map-with-mfs.ll

@@ -58,7 +58,7 @@ declare i32 @qux()
 ; CHECK-LABEL:  .Lfunc_end0:
 
 ; CHECK:                .section        .llvm_bb_addr_map,"o",@llvm_bb_addr_map,.text.hot.foo
-; CHECK-NEXT:   .byte   4               # version
+; CHECK-NEXT:   .byte   5               # version
 ; BASIC-NEXT:   .byte   40              # feature
 ; PGO-NEXT:     .byte   47              # feature
 ; CHECK-NEXT:   .byte   2               # number of basic block ranges

llvm/test/CodeGen/X86/basic-block-address-map.ll

@@ -52,7 +52,7 @@ declare i32 @__gxx_personality_v0(...)
 ; UNIQ:			.section	.llvm_bb_addr_map,"o",@llvm_bb_addr_map,.text._Z3bazb{{$}}
 ;; Verify that with -unique-section-names=false, the unique id of the text section gets assigned to the llvm_bb_addr_map section.
 ; NOUNIQ:		.section	.llvm_bb_addr_map,"o",@llvm_bb_addr_map,.text,unique,1
-; CHECK-NEXT:   .byte   4		# version
+; CHECK-NEXT:   .byte   5		# version
 ; CHECK-NEXT:   .byte   32		# feature
 ; CHECK-NEXT:	.quad	.Lfunc_begin0	# function address
 ; CHECK-NEXT:	.byte	6		# number of basic blocks