Adding Matching and Inference Functionality to Propeller

Author: wdx727

clang/include/clang/Basic/CodeGenOptions.h

@@ -288,6 +288,9 @@ class CodeGenOptions : public CodeGenOptionsBase {
   /// Name of the profile file to use with -fprofile-sample-use.
   std::string SampleProfileFile;
 
+  /// Name of the profile file to use with -fpropeller-profile-use.
+  std::string PropellerProfileFile;
+
   /// Name of the profile file to use as output for with -fmemory-profile.
   std::string MemoryProfileOutput;
 

clang/include/clang/Driver/Options.td

@@ -1672,6 +1672,11 @@ def fprofile_sample_accurate : Flag<["-"], "fprofile-sample-accurate">,
                as cold. Otherwise, treat callsites without profile samples as if
                we have no profile}]>,
    MarshallingInfoFlag<CodeGenOpts<"ProfileSampleAccurate">>;
+def fpropeller_profile_use_EQ : Joined<["-"], "fpropeller-profile-use=">,
+    Group<f_Group>, Visibility<[ClangOption, CC1Option, CLOption]>, 
+    MetaVarName<"<pathname>">,
+    HelpText<"Use propeller profile for profile-guided optimization">,
+    MarshallingInfoString<CodeGenOpts<"PropellerProfileFile">>;
 def fsample_profile_use_profi : Flag<["-"], "fsample-profile-use-profi">,
     Visibility<[ClangOption, CC1Option]>,
     Group<f_Group>,

clang/lib/CodeGen/BackendUtil.cpp

@@ -835,7 +835,8 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
   if (CodeGenOpts.hasProfileIRInstr())
     // -fprofile-generate.
     PGOOpt = PGOOptions(getProfileGenName(CodeGenOpts), "", "",
-                        CodeGenOpts.MemoryProfileUsePath, nullptr,
+                        CodeGenOpts.MemoryProfileUsePath, 
+                        CodeGenOpts.PropellerProfileFile, nullptr,
                         PGOOptions::IRInstr, PGOOptions::NoCSAction,
                         ClPGOColdFuncAttr, CodeGenOpts.DebugInfoForProfiling,
                         /*PseudoProbeForProfiling=*/false,
@@ -846,30 +847,41 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
                                                     : PGOOptions::NoCSAction;
     PGOOpt = PGOOptions(CodeGenOpts.ProfileInstrumentUsePath, "",
                         CodeGenOpts.ProfileRemappingFile,
-                        CodeGenOpts.MemoryProfileUsePath, VFS,
+                        CodeGenOpts.MemoryProfileUsePath, 
+                        CodeGenOpts.PropellerProfileFile, VFS,
                         PGOOptions::IRUse, CSAction, ClPGOColdFuncAttr,
                         CodeGenOpts.DebugInfoForProfiling);
   } else if (!CodeGenOpts.SampleProfileFile.empty())
     // -fprofile-sample-use
     PGOOpt = PGOOptions(
         CodeGenOpts.SampleProfileFile, "", CodeGenOpts.ProfileRemappingFile,
-        CodeGenOpts.MemoryProfileUsePath, VFS, PGOOptions::SampleUse,
+        CodeGenOpts.MemoryProfileUsePath, 
+        CodeGenOpts.PropellerProfileFile, VFS, PGOOptions::SampleUse,
         PGOOptions::NoCSAction, ClPGOColdFuncAttr,
         CodeGenOpts.DebugInfoForProfiling, CodeGenOpts.PseudoProbeForProfiling);
   else if (!CodeGenOpts.MemoryProfileUsePath.empty())
     // -fmemory-profile-use (without any of the above options)
-    PGOOpt = PGOOptions("", "", "", CodeGenOpts.MemoryProfileUsePath, VFS,
+    PGOOpt = PGOOptions("", "", "", CodeGenOpts.MemoryProfileUsePath, 
+                        CodeGenOpts.PropellerProfileFile, VFS,
                         PGOOptions::NoAction, PGOOptions::NoCSAction,
                         ClPGOColdFuncAttr, CodeGenOpts.DebugInfoForProfiling);
+  else if (!CodeGenOpts.PropellerProfileFile.empty())
+    // -fpropeller-profile-use (without any of the above options)
+    PGOOpt = PGOOptions("", "", "", "", 
+                        CodeGenOpts.PropellerProfileFile, VFS,
+                        PGOOptions::NoAction, PGOOptions::NoCSAction,
+                        ClPGOColdFuncAttr, CodeGenOpts.DebugInfoForProfiling);                    
   else if (CodeGenOpts.PseudoProbeForProfiling)
     // -fpseudo-probe-for-profiling
     PGOOpt =
-        PGOOptions("", "", "", /*MemoryProfile=*/"", nullptr,
+        PGOOptions("", "", "", /*MemoryProfile=*/"", 
+                   /*PropellerProfileFile=*/"", nullptr,
                    PGOOptions::NoAction, PGOOptions::NoCSAction,
                    ClPGOColdFuncAttr, CodeGenOpts.DebugInfoForProfiling, true);
   else if (CodeGenOpts.DebugInfoForProfiling)
     // -fdebug-info-for-profiling
-    PGOOpt = PGOOptions("", "", "", /*MemoryProfile=*/"", nullptr,
+    PGOOpt = PGOOptions("", "", "", /*MemoryProfile=*/"", 
+                        /*PropellerProfileFile=*/"", nullptr,
                         PGOOptions::NoAction, PGOOptions::NoCSAction,
                         ClPGOColdFuncAttr, true);
 
@@ -887,7 +899,8 @@ void EmitAssemblyHelper::RunOptimizationPipeline(
       PGOOpt->CSAction = PGOOptions::CSIRInstr;
     } else
       PGOOpt = PGOOptions("", getProfileGenName(CodeGenOpts), "",
-                          /*MemoryProfile=*/"", nullptr, PGOOptions::NoAction,
+                          /*MemoryProfile=*/"", /*PropellerProfileFile=*/"", 
+                          nullptr, PGOOptions::NoAction,
                           PGOOptions::CSIRInstr, ClPGOColdFuncAttr,
                           CodeGenOpts.DebugInfoForProfiling);
   }
@@ -1320,6 +1333,7 @@ runThinLTOBackend(CompilerInstance &CI, ModuleSummaryIndex *CombinedIndex,
   Conf.OptLevel = CGOpts.OptimizationLevel;
   initTargetOptions(CI, Diags, Conf.Options);
   Conf.SampleProfile = std::move(SampleProfile);
+  Conf.PropellerProfile = CGOpts.PropellerProfileFile;
   Conf.PTO.LoopUnrolling = CGOpts.UnrollLoops;
   Conf.PTO.LoopInterchange = CGOpts.InterchangeLoops;
   // For historical reasons, loop interleaving is set to mirror setting for loop

clang/lib/Driver/ToolChains/Clang.cpp

@@ -6416,6 +6416,18 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back("-sample-profile-use-profi");
   }
 
+  if (auto *PropellerUseArg = 
+    Args.getLastArg(options::OPT_fpropeller_profile_use_EQ)) {
+    if (Arg *A = Args.getLastArg(options::OPT_fbasic_block_sections_EQ)) {
+      StringRef Val = A->getValue();
+      if (Val.starts_with("list="))
+        D.Diag(diag::err_drv_argument_not_allowed_with) 
+          << PropellerUseArg->getAsString(Args)
+          << "-fbasic-block-sections=list";
+    }
+    PropellerUseArg->render(Args, CmdArgs);
+  }
+
   // Add runtime flag for PS4/PS5 when PGO, coverage, or sanitizers are enabled.
   if (RawTriple.isPS() &&
       !Args.hasArg(options::OPT_nostdlib, options::OPT_nodefaultlibs)) {

lld/ELF/Config.h

@@ -211,6 +211,7 @@ struct Config {
   llvm::StringRef ltoNewPmPasses;
   llvm::StringRef ltoObjPath;
   llvm::StringRef ltoSampleProfile;
+  llvm::StringRef ltoPropellerProfile;
   llvm::StringRef mapFile;
   llvm::StringRef outputFile;
   llvm::StringRef optRemarksFilename;

lld/ELF/Driver.cpp

@@ -1245,6 +1245,73 @@ static SmallVector<StringRef, 0> getSymbolOrderingFile(Ctx &ctx,
   return names.takeVector();
 }
 
+struct HotBBInfo{
+  uint32_t TotalBBSize = 0;
+  uint32_t HotBBSize = 0;
+};
+static SmallVector<StringRef, 0> generateSymbolOrderingFromPropellerProfile(MemoryBufferRef mb) {
+  SmallVector<std::pair<StringRef, HotBBInfo>> names;
+  SmallVector<HotBBInfo *> hotBBInfos;
+  uint32_t line = 0;
+  for (StringRef s : args::getLines(mb)) {
+    line++;
+    if (!s.consume_front("!") || s.empty()) {
+      error("invalid propeller profile at line: " + Twine(line));
+    }
+    if (s.consume_front("!")) {
+      SmallVector<StringRef, 3> HotBB;
+      s.split(HotBB, ' ');
+      if (HotBB.size() != 3)
+        error("invalid propeller profile at line: " + Twine(line));
+      HotBB[0].consume_front("0x");
+      unsigned long long Hash, Freq, BBID;
+      if (getAsUnsignedInteger(HotBB[0], 16, Hash))
+        error("invalid propeller profile at line: " + Twine(line));
+      if (getAsUnsignedInteger(HotBB[1], 10, Freq))
+        error("invalid propeller profile at line: " + Twine(line));
+      if (getAsUnsignedInteger(HotBB[2], 10, BBID))
+        error("invalid propeller profile at line: " + Twine(line));
+      if (Freq > 0 || BBID == 0) {
+        for (auto hotBBInfo : hotBBInfos) {
+          hotBBInfo->HotBBSize++;
+        }
+      }
+    }
+    else {
+      hotBBInfos.clear();
+      SmallVector<StringRef, 2> NamesWithTotalBBSize;
+      s.split(NamesWithTotalBBSize, ' ');
+      if (NamesWithTotalBBSize.size() != 2)
+        error("invalid propeller profile at line: " + Twine(line));
+      unsigned long long TotalBBSize;
+      if (getAsUnsignedInteger(NamesWithTotalBBSize[1], 10, TotalBBSize))
+        error("invalid propeller profile at line: " + Twine(line));
+      SmallVector<StringRef> funcNames;
+      NamesWithTotalBBSize[0].split(funcNames, '/');
+      for (auto funcName : funcNames) {
+        names.push_back({funcName, {static_cast<uint32_t>(TotalBBSize), 0}});
+        hotBBInfos.push_back(&names.back().second);
+      }
+    }
+  }
+  SmallVector<StringRef, 0> symorders;
+  static std::vector<std::string> storedStrings;
+  for (auto item : names) {
+    if (item.second.HotBBSize > 0) {
+      symorders.push_back(item.first);
+    }
+  }
+  for (auto item : names) {
+    if (item.second.HotBBSize != item.second.TotalBBSize) {
+      std::string str(item.first.str());
+      str.append(".cold");
+      storedStrings.push_back(str);
+      symorders.emplace_back(storedStrings.back());
+    }
+  }
+  return symorders;
+}
+
 static bool getIsRela(Ctx &ctx, opt::InputArgList &args) {
   // The psABI specifies the default relocation entry format.
   bool rela = is_contained({EM_AARCH64, EM_AMDGPU, EM_HEXAGON, EM_LOONGARCH,
@@ -1848,6 +1915,16 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) {
       ctx.arg.symbolOrderingFile = getSymbolOrderingFile(ctx, *buffer);
   }
 
+  if (auto *arg = args.getLastArg(OPT_propeller_profile_use)){
+    if (args.hasArg(OPT_call_graph_ordering_file) || args.hasArg(OPT_symbol_ordering_file))
+      error("--propeller-profile-use and --symbol-ordering-file and --call-graph-order-file "
+            "may not be used together");
+    if (auto buffer = readFile(ctx, arg->getValue())) {
+      ctx.arg.ltoPropellerProfile = arg->getValue();
+      ctx.arg.symbolOrderingFile = generateSymbolOrderingFromPropellerProfile(*buffer);
+    }
+  }
+
   assert(ctx.arg.versionDefinitions.empty());
   ctx.arg.versionDefinitions.push_back(
       {"local", (uint16_t)VER_NDX_LOCAL, {}, {}});

lld/ELF/LTO.cpp

@@ -128,6 +128,7 @@ static lto::Config createConfig(Ctx &ctx) {
   c.StatsFile = std::string(ctx.arg.optStatsFilename);
 
   c.SampleProfile = std::string(ctx.arg.ltoSampleProfile);
+  c.PropellerProfile = std::string(ctx.arg.ltoPropellerProfile);
   for (StringRef pluginFn : ctx.arg.passPlugins)
     c.PassPlugins.push_back(std::string(pluginFn));
   c.DebugPassManager = ctx.arg.ltoDebugPassManager;

lld/ELF/Options.td

@@ -408,6 +408,9 @@ def print_map: F<"print-map">,
 def print_memory_usage: F<"print-memory-usage">,
   HelpText<"Report target memory usage">;
 
+defm propeller_profile_use:
+  EEq<"propeller-profile-use", "Propeller profile file used to reorder symorder">;
+
 defm relax: BB<"relax",
   "Enable target-specific relaxations if supported (default)",
   "Disable target-specific relaxations">;

llvm/include/llvm/CodeGen/FuncHotBBHashesProfileReader.h

@@ -0,0 +1,64 @@
+#ifndef LLVM_CODEGEN_FuncHotBBHashesProfileReader_H
+#define LLVM_CODEGEN_FuncHotBBHashesProfileReader_H
+
+#include "llvm/ADT/SmallString.h"
+#include "llvm/IR/Module.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/MemoryBuffer.h"
+
+namespace llvm {
+
+struct HotBBInfo {
+    uint64_t BBHash;
+    uint64_t Freq;
+};
+
+class FuncHotBBHashesProfileReader : public ImmutablePass {
+public:
+    static char ID;
+
+    FuncHotBBHashesProfileReader(const std::string PropellerProfile);
+
+    FuncHotBBHashesProfileReader();
+
+    StringRef getPassName() const override {
+        return "Basic Block Frequency with Hash Profile Reader";
+    }
+
+    // return a vector of hit BB hashes for a function.
+    std::pair<bool, SmallVector<HotBBInfo, 4>>
+    getHotBBInfosForFunction(StringRef FuncName) const;
+
+    // Reads the profile for matching functions.
+    bool doInitialization(Module &M) override;
+
+private:
+    std::unique_ptr<MemoryBuffer> MBuf;
+
+    StringRef getAliasName(StringRef FuncName) const {
+        auto R = FuncAliasMap.find(FuncName);
+        return R == FuncAliasMap.end() ? FuncName : R->second;
+    }
+
+    // Reads the basic block frequency with hash profile for functions in this module.
+    Error ReadProfile();
+    
+    // Profile file path.
+    std::string PropellerFilePath;
+
+    // Some functions have alias names. We use this map to find the main alias
+    // name for which we have mapping in ProgramBBClusterInfo.
+    StringMap<StringRef> FuncAliasMap;
+
+    // record the frequency of basic block, 
+    // the basic block is represented by its hash.
+    StringMap<SmallVector<HotBBInfo, 4>> FuncToHotBBHashes;
+};
+
+ImmutablePass *
+createFuncHotBBHashesProfileReaderPass(const std::string PropellerProfile);
+
+} // namespace llvm
+
+#endif // LLVM_CODEGEN_FuncHotBBHashesProfileReader_H

llvm/include/llvm/CodeGen/HotMachineBasicBlockInfoGenerator.h

@@ -0,0 +1,51 @@
+#ifndef LLVM_CODEGEN_HotMachineBasicBlockInfoGenerator_H
+#define LLVM_CODEGEN_HotMachineBasicBlockInfoGenerator_H
+
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/Transforms/Utils/SampleProfileInference.h"
+#include "llvm/CodeGen/FuncHotBBHashesProfileReader.h"
+
+namespace llvm {
+
+class HotMachineBasicBlockInfoGenerator : public MachineFunctionPass {
+public:
+  static char ID;
+  HotMachineBasicBlockInfoGenerator();
+
+  StringRef getPassName() const override {
+    return "Basic Block Matching and Inference";
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+
+  bool runOnMachineFunction(MachineFunction &F) override;
+
+  std::optional<SmallVector<MachineBasicBlock *, 4>> getHotMBBs(StringRef FuncName) const;
+
+private:
+  using Edge = std::pair<const MachineBasicBlock *, const MachineBasicBlock *>;
+  using BlockWeightMap = DenseMap<const MachineBasicBlock *, uint64_t>;
+  using EdgeWeightMap = DenseMap<Edge, uint64_t>;
+  using BlockEdgeMap =
+    DenseMap<const MachineBasicBlock *, SmallVector<const MachineBasicBlock *, 8>>;
+
+  DenseMap<StringRef, SmallVector<MachineBasicBlock *, 4>> FuncToHotMBBs;
+    
+  void matchHotBBsByHashes(
+    MachineFunction &MF,
+    SmallVector<HotBBInfo, 4> &HotMBBInfos,
+    BlockWeightMap &MBBToFreq, 
+    BlockEdgeMap &Successors,
+    SmallVector<MachineBasicBlock *, 4> &HotBBs);
+  
+  void generateHotBBsforFunction(
+    MachineFunction &MF,
+    BlockWeightMap &OriBlockWeights,
+    BlockWeightMap &BlockWeights,
+    EdgeWeightMap &EdgeWeights,
+    SmallVector<MachineBasicBlock *, 4> &HotBBs);
+};
+
+} // end namespace llvm
+
+#endif // LLVM_CODEGEN_HotMachineBasicBlockInfoGenerator_H