[𝘀𝗽𝗿] initial version

Created using spr 1.3.4

Author: aaupov

bolt/include/bolt/Core/BinaryContext.h

@@ -760,10 +760,8 @@ class BinaryContext {
     uint32_t NumExactMatchedBlocks{0};
     ///   the number of loosely matched basic blocks
     uint32_t NumLooseMatchedBlocks{0};
-    ///   the number of exactly pseudo probe matched basic blocks
-    uint32_t NumPseudoProbeExactMatchedBlocks{0};
-    ///   the number of loosely pseudo probe matched basic blocks
-    uint32_t NumPseudoProbeLooseMatchedBlocks{0};
+    ///   the number of pseudo probe matched basic blocks
+    uint32_t NumPseudoProbeMatchedBlocks{0};
     ///   the number of call matched basic blocks
     uint32_t NumCallMatchedBlocks{0};
     ///   the total count of samples in the profile
@@ -772,10 +770,8 @@ class BinaryContext {
     uint64_t ExactMatchedSampleCount{0};
     ///   the count of loosely matched samples
     uint64_t LooseMatchedSampleCount{0};
-    ///   the count of exactly pseudo probe matched samples
-    uint64_t PseudoProbeExactMatchedSampleCount{0};
-    ///   the count of loosely pseudo probe matched samples
-    uint64_t PseudoProbeLooseMatchedSampleCount{0};
+    ///   the count of pseudo probe matched samples
+    uint64_t PseudoProbeMatchedSampleCount{0};
     ///   the count of call matched samples
     uint64_t CallMatchedSampleCount{0};
   } Stats;

bolt/include/bolt/Profile/ProfileYAMLMapping.h

@@ -140,6 +140,7 @@ struct BinaryBasicBlockProfile {
   std::vector<CallSiteInfo> CallSites;
   std::vector<SuccessorInfo> Successors;
   std::vector<PseudoProbeInfo> PseudoProbes;
+  std::string PseudoProbesStr;
 
   bool operator==(const BinaryBasicBlockProfile &Other) const {
     return Index == Other.Index;
@@ -163,6 +164,7 @@ template <> struct MappingTraits<bolt::BinaryBasicBlockProfile> {
                        std::vector<bolt::SuccessorInfo>());
     YamlIO.mapOptional("probes", BBP.PseudoProbes,
                        std::vector<bolt::PseudoProbeInfo>());
+    YamlIO.mapOptional("probe", BBP.PseudoProbesStr, std::string());
   }
 };
 
@@ -207,6 +209,7 @@ struct BinaryFunctionProfile {
   uint64_t ExternEntryCount{0};
   std::vector<BinaryBasicBlockProfile> Blocks;
   std::vector<InlineTreeNode> InlineTree;
+  std::string InlineTreeStr;
   bool Used{false};
 };
 } // end namespace bolt
@@ -223,6 +226,7 @@ template <> struct MappingTraits<bolt::BinaryFunctionProfile> {
                        std::vector<bolt::BinaryBasicBlockProfile>());
     YamlIO.mapOptional("inline_tree", BFP.InlineTree,
                        std::vector<bolt::InlineTreeNode>());
+    YamlIO.mapOptional("ppit", BFP.InlineTreeStr, std::string());
   }
 };
 

bolt/include/bolt/Profile/YAMLProfileReader.h

@@ -45,9 +45,6 @@ class YAMLProfileReader : public ProfileReaderBase {
   using ProfileLookupMap =
       DenseMap<uint32_t, yaml::bolt::BinaryFunctionProfile *>;
 
-  using GUIDInlineTreeMap =
-      std::unordered_map<uint64_t, const MCDecodedPseudoProbeInlineTree *>;
-
   /// A class for matching binary functions in functions in the YAML profile.
   /// First, a call graph is constructed for both profiled and binary functions.
   /// Then functions are hashed based on the names of their callee/caller
@@ -110,51 +107,40 @@ class YAMLProfileReader : public ProfileReaderBase {
   //     (profile)                             (binary)
   //      | blocks                                ^
   //      v                                       |
-  // yaml::bolt::BinaryBasicBlockProfile ~= FlowBlock
-  //     ||| probes                               ^  (majority vote)
-  //      v                                      ||| BBPseudoProbeToBlock
-  // yaml::bolt::PseudoProbeInfo            MCDecodedPseudoProbe
+  // BinaryBasicBlockProfile                BasicBlock
+  //      | probes                                ^
+  //      v                                       | address
+  // PseudoProbeInfo                        MCDecodedPseudoProbe
   //      | InlineTreeIndex                       ^
   //      v                                       | probe id
   // [ profile node id (uint32_t)    ->     MCDecodedPseudoProbeInlineTree *]
   //                     InlineTreeNodeMapTy
-  class InlineTreeNodeMapTy {
-    DenseMap<uint32_t, const MCDecodedPseudoProbeInlineTree *> Map;
-
-    void mapInlineTreeNode(uint32_t ProfileNodeIdx,
-                           const MCDecodedPseudoProbeInlineTree *BinaryNode) {
-      auto Res = Map.try_emplace(ProfileNodeIdx, BinaryNode);
-      assert(Res.second &&
-             "Duplicate mapping from profile node index to binary inline tree");
-      (void)Res;
-    }
-
-  public:
-    /// Returns matched InlineTree * for a given profile inline_tree_id.
-    const MCDecodedPseudoProbeInlineTree *
-    getInlineTreeNode(uint32_t ProfileInlineTreeNodeId) const {
-      auto It = Map.find(ProfileInlineTreeNodeId);
-      if (It == Map.end())
-        return nullptr;
-      return It->second;
-    }
+  using InlineTreeNodeMapTy =
+      std::vector<const MCDecodedPseudoProbeInlineTree *>;
 
-    // Match up \p YamlInlineTree with binary inline tree rooted at \p Root.
-    // Return the number of matched nodes.
-    //
-    // This function populates the mapping from profile inline tree node id to a
-    // corresponding binary MCDecodedPseudoProbeInlineTree node.
-    size_t matchInlineTrees(
-        const MCPseudoProbeDecoder &Decoder,
-        const std::vector<yaml::bolt::InlineTreeNode> &YamlInlineTree,
-        const MCDecodedPseudoProbeInlineTree *Root);
-  };
-
-  // Partial probe matching specification: matched inline tree and corresponding
-  // BinaryFunctionProfile
+  // Match up \p YamlInlineTree with binary inline tree rooted at \p Root.
+  // Return the number of matched nodes.
+  //
+  // This function populates the \p Map indexed by \p YamlInlineTree from
+  // profile inline tree node id to a corresponding binary
+  // MCDecodedPseudoProbeInlineTree node on a successful match or nullptr.
+  size_t matchInlineTreesImpl(
+      BinaryFunction &BF, yaml::bolt::BinaryFunctionProfile &YamlBF,
+      const MCDecodedPseudoProbeInlineTree &Root, uint32_t RootIdx,
+      ArrayRef<yaml::bolt::InlineTreeNode> YamlInlineTree,
+      MutableArrayRef<const MCDecodedPseudoProbeInlineTree *> Map, float Scale);
+
+  /// Support mapping the profile to the same binary function more than once.
+  /// The index is used to check and prevent recursive (backtracking) matches.
+  /// The value is a node map and scale factor, used to downscale outlined
+  /// profile when inlining it into the function.
+  using RootIdxToMapTy =
+      std::map<uint32_t, std::pair<InlineTreeNodeMapTy, float>>;
+  /// Probe matching specification: map from \p BinaryFunctionProfile to the
+  /// partial match specification \p RootIdxToMapTy.
   using ProbeMatchSpec =
-      std::pair<InlineTreeNodeMapTy,
-                std::reference_wrapper<yaml::bolt::BinaryFunctionProfile>>;
+      std::unordered_map<const yaml::bolt::BinaryFunctionProfile *,
+                         RootIdxToMapTy>;
 
 private:
   /// Adjustments for basic samples profiles (without LBR).
@@ -183,18 +169,43 @@ class YAMLProfileReader : public ProfileReaderBase {
   /// Map a common LTO prefix to a set of binary functions.
   StringMap<std::unordered_set<BinaryFunction *>> LTOCommonNameFunctionMap;
 
+  /// For pseudo probe function matching.
+  /// Set of profile GUIDs.
+  std::unordered_set<uint64_t> YamlGUIDs;
+  /// Map binary function GUID to binary function.
+  std::unordered_multimap<uint64_t, BinaryFunction *> GUIDToBF;
+
   /// Function names in profile.
   StringSet<> ProfileFunctionNames;
 
   /// BinaryFunction pointers indexed by YamlBP functions.
   std::vector<BinaryFunction *> ProfileBFs;
 
-  // Pseudo probe function GUID to inline tree node
-  GUIDInlineTreeMap TopLevelGUIDToInlineTree;
-
   // Mapping from a binary function to its partial match specification
   // (YAML profile and its inline tree mapping to binary).
-  DenseMap<BinaryFunction *, std::vector<ProbeMatchSpec>> BFToProbeMatchSpecs;
+  DenseMap<BinaryFunction *, ProbeMatchSpec> BFToProbeMatchSpecs;
+
+  /// Pseudo probe matching stats.
+  struct {
+    // Inline tree root mismatch causes.
+    uint64_t MismatchingRootGUID{0};
+    uint64_t MismatchingRootHash{0};
+    // Inline tree internal node mismatch causes.
+    uint64_t MismatchingNodeHash{0};
+    uint64_t MissingProfileNode{0};
+    // Incomplete profile causes.
+    uint64_t MissingCallProbe{0};
+    uint64_t MissingCallee{0};
+    uint64_t MissingInlineTree{0};
+    uint64_t TotalCallSites{0};
+    uint64_t MissingCallCount{0};
+    uint64_t TotalCallCount{0};
+    // Total matched stats.
+    uint64_t MatchedRoots{0};
+    uint64_t MatchedNodes{0};
+    uint64_t AttemptedNodes{0};
+    uint64_t AttemptedRoots{0};
+  } ProbeMatchingStats;
 
   /// Populate \p Function profile with the one supplied in YAML format.
   bool parseFunctionProfile(BinaryFunction &Function,
@@ -207,7 +218,7 @@ class YAMLProfileReader : public ProfileReaderBase {
   /// Infer function profile from stale data (collected on older binaries).
   bool inferStaleProfile(BinaryFunction &Function,
                          const yaml::bolt::BinaryFunctionProfile &YamlBF,
-                         const ArrayRef<ProbeMatchSpec> ProbeMatchSpecs);
+                         const ProbeMatchSpec &ProbeMatchSpecs);
 
   /// Initialize maps for profile matching.
   void buildNameMaps(BinaryContext &BC);
@@ -242,6 +253,18 @@ class YAMLProfileReader : public ProfileReaderBase {
     ProfiledFunctions.emplace(&BF);
   }
 
+  /// Return a top-level binary inline tree node for a given \p BF
+  const MCDecodedPseudoProbeInlineTree *
+  lookupTopLevelNode(const BinaryFunction &BF);
+
+  /// Match up \p BF binary inline trees starting at root or \p Node if set, and
+  /// \p YamlBF profile starting at \p NodeIdx and record the mapping in
+  /// BFToProbeMatchSpecs.
+  void matchInlineTrees(BinaryFunction &BF,
+                        const MCDecodedPseudoProbeInlineTree *Node,
+                        yaml::bolt::BinaryFunctionProfile &YamlBF,
+                        uint32_t NodeIdx, float Scale);
+
   /// Check if the profile uses an event with a given \p Name.
   bool usesEvent(StringRef Name) const;
 };

bolt/include/bolt/Profile/YAMLProfileWriter.h

@@ -36,16 +36,14 @@ class YAMLProfileWriter {
       DenseMap<const MCDecodedPseudoProbeInlineTree *, uint32_t>;
   struct InlineTreeDesc {
     template <typename T> using GUIDMapTy = std::unordered_map<uint64_t, T>;
-    using GUIDNodeMap = GUIDMapTy<const MCDecodedPseudoProbeInlineTree *>;
     using GUIDNumMap = GUIDMapTy<uint32_t>;
-    GUIDNodeMap TopLevelGUIDToInlineTree;
     GUIDNumMap GUIDIdxMap;
     GUIDNumMap HashIdxMap;
   };
 
-  static std::tuple<std::vector<yaml::bolt::InlineTreeNode>, InlineTreeMapTy>
-  convertBFInlineTree(const MCPseudoProbeDecoder &Decoder,
-                      const InlineTreeDesc &InlineTree, uint64_t GUID);
+  static InlineTreeMapTy convertBFInlineTree(
+      const MCPseudoProbeDecoder &Decoder, const InlineTreeDesc &InlineTree,
+      const BinaryFunction &BF, yaml::bolt::BinaryFunctionProfile &YamlBF);
 
   static std::tuple<yaml::bolt::ProfilePseudoProbeDesc, InlineTreeDesc>
   convertPseudoProbeDesc(const MCPseudoProbeDecoder &PseudoProbeDecoder);

bolt/include/bolt/Utils/CommandLineOpts.h

@@ -134,6 +134,7 @@ enum GadgetScannerKind { GS_PACRET, GS_PAUTH, GS_ALL };
 
 extern llvm::cl::bits<GadgetScannerKind> GadgetScannersToRun;
 
+enum ProbesWriteMode : char { PWM_None = 0, PWM_Default, PWM_Compact };
 } // namespace opts
 
 namespace llvm {

bolt/lib/Passes/BinaryPasses.cpp

@@ -1555,25 +1555,14 @@ Error PrintProgramStats::runOnFunctions(BinaryContext &BC) {
         100.0 * BC.Stats.ExactMatchedSampleCount / BC.Stats.StaleSampleCount,
         BC.Stats.ExactMatchedSampleCount, BC.Stats.StaleSampleCount);
     BC.outs() << format(
-        "BOLT-INFO: inference found an exact pseudo probe match for %.2f%% of "
+        "BOLT-INFO: inference found pseudo probe match for %.2f%% of "
         "basic blocks (%zu out of %zu stale) responsible for %.2f%% samples"
         " (%zu out of %zu stale)\n",
-        100.0 * BC.Stats.NumPseudoProbeExactMatchedBlocks /
-            BC.Stats.NumStaleBlocks,
-        BC.Stats.NumPseudoProbeExactMatchedBlocks, BC.Stats.NumStaleBlocks,
-        100.0 * BC.Stats.PseudoProbeExactMatchedSampleCount /
+        100.0 * BC.Stats.NumPseudoProbeMatchedBlocks / BC.Stats.NumStaleBlocks,
+        BC.Stats.NumPseudoProbeMatchedBlocks, BC.Stats.NumStaleBlocks,
+        100.0 * BC.Stats.PseudoProbeMatchedSampleCount /
             BC.Stats.StaleSampleCount,
-        BC.Stats.PseudoProbeExactMatchedSampleCount, BC.Stats.StaleSampleCount);
-    BC.outs() << format(
-        "BOLT-INFO: inference found a loose pseudo probe match for %.2f%% of "
-        "basic blocks (%zu out of %zu stale) responsible for %.2f%% samples"
-        " (%zu out of %zu stale)\n",
-        100.0 * BC.Stats.NumPseudoProbeLooseMatchedBlocks /
-            BC.Stats.NumStaleBlocks,
-        BC.Stats.NumPseudoProbeLooseMatchedBlocks, BC.Stats.NumStaleBlocks,
-        100.0 * BC.Stats.PseudoProbeLooseMatchedSampleCount /
-            BC.Stats.StaleSampleCount,
-        BC.Stats.PseudoProbeLooseMatchedSampleCount, BC.Stats.StaleSampleCount);
+        BC.Stats.PseudoProbeMatchedSampleCount, BC.Stats.StaleSampleCount);
     BC.outs() << format(
         "BOLT-INFO: inference found a call match for %.2f%% of basic "
         "blocks"

bolt/lib/Profile/DataAggregator.cpp

@@ -114,7 +114,7 @@ MaxSamples("max-samples",
   cl::cat(AggregatorCategory));
 
 extern cl::opt<opts::ProfileFormatKind> ProfileFormat;
-extern cl::opt<bool> ProfileWritePseudoProbes;
+extern cl::opt<ProbesWriteMode> ProfileWritePseudoProbes;
 extern cl::opt<std::string> SaveProfile;
 
 cl::opt<bool> ReadPreAggregated(
@@ -2388,9 +2388,8 @@ std::error_code DataAggregator::writeBATYAML(BinaryContext &BC,
         DenseMap<const MCDecodedPseudoProbeInlineTree *, uint32_t>
             InlineTreeNodeId;
         if (BF->getGUID()) {
-          std::tie(YamlBF.InlineTree, InlineTreeNodeId) =
-              YAMLProfileWriter::convertBFInlineTree(*PseudoProbeDecoder,
-                                                     InlineTree, BF->getGUID());
+          InlineTreeNodeId = YAMLProfileWriter::convertBFInlineTree(
+              *PseudoProbeDecoder, InlineTree, *BF, YamlBF);
         }
         // Fetch probes belonging to all fragments
         const AddressProbesMap &ProbeMap =