Update with #107137 changes

Created using spr 1.3.4

Author: aaupov

bolt/include/bolt/Profile/ProfileYAMLMapping.h

@@ -96,9 +96,8 @@ template <> struct MappingTraits<bolt::SuccessorInfo> {
 namespace bolt {
 struct PseudoProbeInfo {
   uint32_t InlineTreeIndex = 0;
-  uint64_t BlockMask = 0; // bitset with probe indices
-  // Assume BlockMask == 1 if no other probes are set
-  std::vector<uint64_t> BlockProbes;
+  uint64_t BlockMask = 0; // bitset with probe indices from 1 to 64
+  std::vector<uint64_t> BlockProbes; // block probes with indices above 64
   std::vector<uint64_t> CallProbes;
   std::vector<uint64_t> IndCallProbes;
   std::vector<uint32_t> InlineTreeNodes;
@@ -113,10 +112,10 @@ struct PseudoProbeInfo {
 
 template <> struct MappingTraits<bolt::PseudoProbeInfo> {
   static void mapping(IO &YamlIO, bolt::PseudoProbeInfo &PI) {
-    YamlIO.mapOptional("blk", PI.BlockMask, 0);
-    YamlIO.mapOptional("blks", PI.BlockProbes, std::vector<uint64_t>());
-    YamlIO.mapOptional("calls", PI.CallProbes, std::vector<uint64_t>());
-    YamlIO.mapOptional("indcalls", PI.IndCallProbes, std::vector<uint64_t>());
+    YamlIO.mapOptional("blx", PI.BlockMask, 0);
+    YamlIO.mapOptional("blk", PI.BlockProbes, std::vector<uint64_t>());
+    YamlIO.mapOptional("call", PI.CallProbes, std::vector<uint64_t>());
+    YamlIO.mapOptional("icall", PI.IndCallProbes, std::vector<uint64_t>());
     YamlIO.mapOptional("id", PI.InlineTreeIndex, 0);
     YamlIO.mapOptional("ids", PI.InlineTreeNodes, std::vector<uint32_t>());
   }
@@ -170,18 +169,18 @@ template <> struct MappingTraits<bolt::BinaryBasicBlockProfile> {
 };
 
 namespace bolt {
-struct InlineTreeInfo {
+struct InlineTreeNode {
   uint32_t ParentIndexDelta;
   uint32_t CallSiteProbe;
-  // Index in PseudoProbeDesc.GUID + 1, 0 for same as previous
+  // Index in PseudoProbeDesc.GUID, UINT32_MAX for same as previous (omitted)
   uint32_t GUIDIndex;
-  bool operator==(const InlineTreeInfo &) const { return false; }
+  bool operator==(const InlineTreeNode &) const { return false; }
 };
 } // end namespace bolt
 
-template <> struct MappingTraits<bolt::InlineTreeInfo> {
-  static void mapping(IO &YamlIO, bolt::InlineTreeInfo &ITI) {
-    YamlIO.mapOptional("g", ITI.GUIDIndex, 0);
+template <> struct MappingTraits<bolt::InlineTreeNode> {
+  static void mapping(IO &YamlIO, bolt::InlineTreeNode &ITI) {
+    YamlIO.mapOptional("g", ITI.GUIDIndex, UINT32_MAX);
     YamlIO.mapOptional("p", ITI.ParentIndexDelta, 0);
     YamlIO.mapOptional("cs", ITI.CallSiteProbe, 0);
   }
@@ -192,7 +191,7 @@ template <> struct MappingTraits<bolt::InlineTreeInfo> {
 } // end namespace llvm
 
 LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::bolt::BinaryBasicBlockProfile)
-LLVM_YAML_IS_FLOW_SEQUENCE_VECTOR(llvm::yaml::bolt::InlineTreeInfo)
+LLVM_YAML_IS_FLOW_SEQUENCE_VECTOR(llvm::yaml::bolt::InlineTreeNode)
 
 namespace llvm {
 namespace yaml {
@@ -205,7 +204,7 @@ struct BinaryFunctionProfile {
   llvm::yaml::Hex64 Hash{0};
   uint64_t ExecCount{0};
   std::vector<BinaryBasicBlockProfile> Blocks;
-  std::vector<InlineTreeInfo> InlineTree;
+  std::vector<InlineTreeNode> InlineTree;
   bool Used{false};
 };
 } // end namespace bolt
@@ -220,7 +219,7 @@ template <> struct MappingTraits<bolt::BinaryFunctionProfile> {
     YamlIO.mapOptional("blocks", BFP.Blocks,
                        std::vector<bolt::BinaryBasicBlockProfile>());
     YamlIO.mapOptional("inline_tree", BFP.InlineTree,
-                       std::vector<bolt::InlineTreeInfo>());
+                       std::vector<bolt::InlineTreeNode>());
   }
 };
 
@@ -274,20 +273,21 @@ namespace bolt {
 struct PseudoProbeDesc {
   std::vector<Hex64> GUID;
   std::vector<Hex64> Hash;
-  std::vector<uint32_t> GUIDHash; // Index of hash for that GUID in Hash
+  std::vector<uint32_t> GUIDHashIdx; // Index of hash for that GUID in Hash
 
   bool operator==(const PseudoProbeDesc &Other) const {
     // Only treat empty Desc as equal
     return GUID.empty() && Other.GUID.empty() && Hash.empty() &&
-           Other.Hash.empty() && GUIDHash.empty() && Other.GUIDHash.empty();
+           Other.Hash.empty() && GUIDHashIdx.empty() &&
+           Other.GUIDHashIdx.empty();
   }
 };
 } // end namespace bolt
 
 template <> struct MappingTraits<bolt::PseudoProbeDesc> {
   static void mapping(IO &YamlIO, bolt::PseudoProbeDesc &PD) {
     YamlIO.mapRequired("gs", PD.GUID);
-    YamlIO.mapRequired("gh", PD.GUIDHash);
+    YamlIO.mapRequired("gh", PD.GUIDHashIdx);
     YamlIO.mapRequired("hs", PD.Hash);
   }
 };

bolt/include/bolt/Profile/YAMLProfileWriter.h

@@ -43,18 +43,18 @@ class YAMLProfileWriter {
     GUIDNumMap HashIdxMap;
   };
 
-  static std::tuple<std::vector<yaml::bolt::InlineTreeInfo>, InlineTreeMapTy>
+  static std::tuple<std::vector<yaml::bolt::InlineTreeNode>, InlineTreeMapTy>
   convertBFInlineTree(const MCPseudoProbeDecoder &Decoder,
                       const InlineTreeDesc &InlineTree, uint64_t GUID);
 
+  static std::tuple<yaml::bolt::PseudoProbeDesc, InlineTreeDesc>
+  convertPseudoProbeDesc(const MCPseudoProbeDecoder &PseudoProbeDecoder);
+
   static yaml::bolt::BinaryFunctionProfile
   convert(const BinaryFunction &BF, bool UseDFS,
           const InlineTreeDesc &InlineTree,
           const BoltAddressTranslation *BAT = nullptr);
 
-  static std::tuple<yaml::bolt::PseudoProbeDesc, InlineTreeDesc>
-  convertPseudoProbeDesc(const MCPseudoProbeDecoder &PseudoProbeDecoder);
-
   /// Set CallSiteInfo destination fields from \p Symbol and return a target
   /// BinaryFunction for that symbol.
   static const BinaryFunction *
@@ -71,8 +71,8 @@ class YAMLProfileWriter {
     uint32_t InlineSite;
   };
   static std::vector<InlineTreeNode>
-  getInlineTree(const MCPseudoProbeDecoder &Decoder,
-                const MCDecodedPseudoProbeInlineTree *Root);
+  collectInlineTree(const MCPseudoProbeDecoder &Decoder,
+                    const MCDecodedPseudoProbeInlineTree &Root);
 
   // 0 - block probe, 1 - indirect call, 2 - direct call
   using ProbeList = std::array<SmallVector<uint64_t, 0>, 3>;

bolt/lib/Profile/StaleProfileMatching.cpp

@@ -233,7 +233,7 @@ class StaleMatcher {
   std::pair<const FlowBlock *, MatchMethod>
   matchBlock(BlendedBlockHash BlendedHash, uint64_t CallHash,
              const ArrayRef<yaml::bolt::PseudoProbeInfo> PseudoProbes,
-             const ArrayRef<yaml::bolt::InlineTreeInfo> InlineTree) {
+             const ArrayRef<yaml::bolt::InlineTreeNode> InlineTree) {
     const auto &[Block, ExactHash] = matchWithOpcodes(BlendedHash);
     if (Block && ExactHash)
       return {Block, MATCH_EXACT};
@@ -327,7 +327,7 @@ class StaleMatcher {
   /// unambiguous.
   std::pair<const FlowBlock *, bool> matchWithPseudoProbes(
       const ArrayRef<yaml::bolt::PseudoProbeInfo> BlockPseudoProbes,
-      const ArrayRef<yaml::bolt::InlineTreeInfo> InlineTree) const {
+      const ArrayRef<yaml::bolt::InlineTreeNode> InlineTree) const {
     if (!opts::StaleMatchingWithPseudoProbes)
       return {nullptr, false};
 
@@ -353,11 +353,7 @@ class StaleMatcher {
     };
 
     auto matchProbe = [&](const yaml::bolt::PseudoProbeInfo &Probe,
-                          uint32_t NodeId, bool IsBlock1) {
-      if (IsBlock1) {
-        ++FlowBlockMatchCount[match(NodeId, 1)];
-        return;
-      }
+                          uint32_t NodeId) {
       for (uint64_t Index = 0; Index < 64; ++Index)
         if (Probe.BlockMask & 1ull << Index)
           ++FlowBlockMatchCount[match(NodeId, Index + 1)];
@@ -368,13 +364,11 @@ class StaleMatcher {
     };
 
     for (const yaml::bolt::PseudoProbeInfo &Probe : BlockPseudoProbes) {
-      bool IsBlock1 = Probe.BlockMask == 0 && Probe.BlockProbes.empty() &&
-                      Probe.IndCallProbes.empty() && Probe.CallProbes.empty();
       if (!Probe.InlineTreeNodes.empty())
         for (uint32_t Node : Probe.InlineTreeNodes)
-          matchProbe(Probe, Node, IsBlock1);
+          matchProbe(Probe, Node);
       else
-        matchProbe(Probe, Probe.InlineTreeIndex, IsBlock1);
+        matchProbe(Probe, Probe.InlineTreeIndex);
     }
     uint32_t BestMatchCount = 0;
     uint32_t TotalMatchCount = 0;
@@ -622,16 +616,16 @@ size_t matchWeightsByHashes(
     uint32_t ParentId = 0;
     uint32_t PrevGUIDIdx = 0;
     uint32_t Index = 0;
-    for (const yaml::bolt::InlineTreeInfo &InlineTreeNode : YamlBF.InlineTree) {
+    for (const yaml::bolt::InlineTreeNode &InlineTreeNode : YamlBF.InlineTree) {
       uint64_t GUIDIdx = InlineTreeNode.GUIDIndex;
-      if (GUIDIdx)
+      if (GUIDIdx != UINT32_MAX)
         PrevGUIDIdx = GUIDIdx;
       else
         GUIDIdx = PrevGUIDIdx;
-      assert(GUIDIdx - 1 < YamlPD.GUID.size());
-      assert(GUIDIdx - 1 < YamlPD.GUIDHash.size());
-      uint64_t GUID = YamlPD.GUID[GUIDIdx - 1];
-      uint32_t HashIdx = YamlPD.GUIDHash[GUIDIdx - 1];
+      assert(GUIDIdx < YamlPD.GUID.size());
+      assert(GUIDIdx < YamlPD.GUIDHashIdx.size());
+      uint64_t GUID = YamlPD.GUID[GUIDIdx];
+      uint32_t HashIdx = YamlPD.GUIDHashIdx[GUIDIdx];
       assert(HashIdx < YamlPD.Hash.size());
       uint64_t Hash = YamlPD.Hash[HashIdx];
       uint32_t InlineTreeNodeId = Index++;

bolt/lib/Profile/YAMLProfileWriter.cpp

@@ -18,7 +18,6 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/raw_ostream.h"
-#include <queue>
 
 #undef  DEBUG_TYPE
 #define DEBUG_TYPE "bolt-prof"
@@ -61,23 +60,21 @@ const BinaryFunction *YAMLProfileWriter::setCSIDestination(
 }
 
 std::vector<YAMLProfileWriter::InlineTreeNode>
-YAMLProfileWriter::getInlineTree(const MCPseudoProbeDecoder &Decoder,
-                                 const MCDecodedPseudoProbeInlineTree *Root) {
+YAMLProfileWriter::collectInlineTree(
+    const MCPseudoProbeDecoder &Decoder,
+    const MCDecodedPseudoProbeInlineTree &Root) {
   auto getHash = [&](const MCDecodedPseudoProbeInlineTree &Node) {
     return Decoder.getFuncDescForGUID(Node.Guid)->FuncHash;
   };
-  assert(Root);
-  std::vector<InlineTreeNode> InlineTree;
-  InlineTreeNode Node{Root, Root->Guid, getHash(*Root), 0, 0};
-  InlineTree.emplace_back(Node);
+  std::vector<InlineTreeNode> InlineTree(
+      {InlineTreeNode{&Root, Root.Guid, getHash(Root), 0, 0}});
   uint32_t ParentId = 0;
   while (ParentId != InlineTree.size()) {
     const MCDecodedPseudoProbeInlineTree *Cur = InlineTree[ParentId].InlineTree;
-    for (const MCDecodedPseudoProbeInlineTree &Child : Cur->getChildren()) {
-      InlineTreeNode Node{&Child, Child.Guid, getHash(Child), ParentId,
-                          std::get<1>(Child.getInlineSite())};
-      InlineTree.emplace_back(Node);
-    }
+    for (const MCDecodedPseudoProbeInlineTree &Child : Cur->getChildren())
+      InlineTree.emplace_back(
+          InlineTreeNode{&Child, Child.Guid, getHash(Child), ParentId,
+                         std::get<1>(Child.getInlineSite())});
     ++ParentId;
   }
 
@@ -125,7 +122,7 @@ YAMLProfileWriter::convertPseudoProbeDesc(const MCPseudoProbeDecoder &Decoder) {
     Desc.GUID.emplace_back(GUID);
     InlineTree.GUIDIdxMap[GUID] = Index++;
     uint64_t Hash = Decoder.getFuncDescForGUID(GUID)->FuncHash;
-    Desc.GUIDHash.emplace_back(InlineTree.HashIdxMap[Hash]);
+    Desc.GUIDHashIdx.emplace_back(InlineTree.HashIdxMap[Hash]);
   }
 
   return {Desc, InlineTree};
@@ -176,39 +173,35 @@ YAMLProfileWriter::convertNodeProbes(NodeIdToProbes &NodeProbes) {
     else
       YamlBPI.InlineTreeNodes = Nodes;
     handleMask(BPI.BlockProbes, YamlBPI.BlockProbes, YamlBPI.BlockMask);
-    // Assume BlockMask == 1 if no other probes are set
-    if (YamlBPI.BlockMask == 1 && YamlBPI.CallProbes.empty() &&
-        YamlBPI.IndCallProbes.empty())
-      YamlBPI.BlockMask = 0;
   }
   return YamlProbes;
 }
 
-std::tuple<std::vector<yaml::bolt::InlineTreeInfo>,
+std::tuple<std::vector<yaml::bolt::InlineTreeNode>,
            YAMLProfileWriter::InlineTreeMapTy>
 YAMLProfileWriter::convertBFInlineTree(const MCPseudoProbeDecoder &Decoder,
                                        const InlineTreeDesc &InlineTree,
                                        uint64_t GUID) {
   DenseMap<const MCDecodedPseudoProbeInlineTree *, uint32_t> InlineTreeNodeId;
-  std::vector<yaml::bolt::InlineTreeInfo> YamlInlineTree;
+  std::vector<yaml::bolt::InlineTreeNode> YamlInlineTree;
   auto It = InlineTree.TopLevelGUIDToInlineTree.find(GUID);
   if (It == InlineTree.TopLevelGUIDToInlineTree.end())
     return {YamlInlineTree, InlineTreeNodeId};
   const MCDecodedPseudoProbeInlineTree *Root = It->second;
-  assert(Root);
+  assert(Root && "Malformed TopLevelGUIDToInlineTree");
   uint32_t Index = 0;
   uint32_t PrevParent = 0;
   uint32_t PrevGUIDIdx = 0;
-  for (const auto &Node : getInlineTree(Decoder, Root)) {
+  for (const auto &Node : collectInlineTree(Decoder, *Root)) {
     InlineTreeNodeId[Node.InlineTree] = Index++;
     auto GUIDIdxIt = InlineTree.GUIDIdxMap.find(Node.GUID);
-    assert(GUIDIdxIt != InlineTree.GUIDIdxMap.end());
-    uint32_t GUIDIdx = GUIDIdxIt->second + 1;
+    assert(GUIDIdxIt != InlineTree.GUIDIdxMap.end() && "Malformed GUIDIdxMap");
+    uint32_t GUIDIdx = GUIDIdxIt->second;
     if (GUIDIdx == PrevGUIDIdx)
-      GUIDIdx = 0;
+      GUIDIdx = UINT32_MAX;
     else
       PrevGUIDIdx = GUIDIdx;
-    YamlInlineTree.emplace_back(yaml::bolt::InlineTreeInfo{
+    YamlInlineTree.emplace_back(yaml::bolt::InlineTreeNode{
         Node.ParentId - PrevParent, Node.InlineSite, GUIDIdx});
     PrevParent = Node.ParentId;
   }

bolt/test/X86/match-blocks-with-pseudo-probes.test

@@ -55,8 +55,8 @@ functions:
         hash:            0xFFFFFFFFFFFFFFF1
         insns:           1
         succ:            [ { bid: 3, cnt: 1} ]
-        probes:          [ { } ]
-    inline_tree:         [ { g: 1 } ]
+        probes:          [ { blx: 1 } ]
+    inline_tree:         [ { g: 0 } ]
 pseudo_probe_desc:
   gs:              [ 0xDB956436E78DD5FA ]
   gh:              [ 0 ]

bolt/test/X86/pseudoprobe-decoding-inline.test

@@ -14,18 +14,18 @@
 # RUN: FileCheck --input-file %t.yaml2 %s --check-prefix CHECK-YAML
 # CHECK-YAML: name: bar
 # CHECK-YAML: - bid: 0
-# CHECK-YAML:   probes: [ { blk: 9 } ]
-# CHECK-YAML: inline_tree: [ { g: 1 } ]
+# CHECK-YAML:   probes: [ { blx: 9 } ]
+# CHECK-YAML: inline_tree: [ { } ]
 #
 # CHECK-YAML: name: foo
 # CHECK-YAML: - bid: 0
-# CHECK-YAML:   probes: [ { blk: 3 } ]
-# CHECK-YAML: inline_tree: [ { g: 2 }, { g: 1, cs: 8 } ]
+# CHECK-YAML:   probes: [ { blx: 3 } ]
+# CHECK-YAML: inline_tree: [ { g: 1 }, { g: 0, cs: 8 } ]
 #
 # CHECK-YAML: name: main
 # CHECK-YAML: - bid: 0
-# CHECK-YAML:   probes: [ { blk: 3, id: 1 }, {  } ]
-# CHECK-YAML: inline_tree: [ { g: 3 }, { g: 2, cs: 2 }, { g: 1, p: 1, cs: 8 } ]
+# CHECK-YAML:   probes: [ { blx: 3, id: 1 }, { blx: 1 } ]
+# CHECK-YAML: inline_tree: [ { g: 2 }, { g: 1, cs: 2 }, { g: 0, p: 1, cs: 8 } ]
 #
 # CHECK-YAML: pseudo_probe_desc:
 # CHECK-YAML-NEXT: gs: [ 0xE413754A191DB537, 0x5CF8C24CDB18BDAC, 0xDB956436E78DD5FA ]