[PERF2BOLT] Check build-ids of binaries when aggregating

Summary:
Check the build-id of the input binary against the build-id of
the binary used during profiling data collection with perf, as reported
in perf.data. If they differ, issue a warning, since the user should use
exactly the same binary. If we cannot determine the build-id of either
the input binary or the one registered in the input perf.data, cancel the
build-id check but print a log message.

(cherry picked from FBD6001917)

Author: rafaelauler

bolt/DataAggregator.cpp

@@ -60,14 +60,25 @@ void DataAggregator::findPerfExecutable() {
 
 void DataAggregator::start(StringRef PerfDataFilename) {
   Enabled = true;
+  this->PerfDataFilename = PerfDataFilename;
   outs() << "PERF2BOLT: Starting data aggregation job for " << PerfDataFilename
          << "\n";
   findPerfExecutable();
-  launchPerfEventsNoWait(PerfDataFilename);
-  launchPerfTasksNoWait(PerfDataFilename);
+  launchPerfEventsNoWait();
+  launchPerfTasksNoWait();
 }
 
-bool DataAggregator::launchPerfEventsNoWait(StringRef PerfDataFilename) {
+void DataAggregator::abort() {
+  std::string Error;
+
+  // Kill subprocesses in case they are not finished
+  sys::Wait(TasksPI, 1, false, &Error);
+  sys::Wait(EventsPI, 1, false, &Error);
+
+  deleteTempFiles();
+}
+
+bool DataAggregator::launchPerfEventsNoWait() {
   SmallVector<const char*, 4> Argv;
   SmallVector<StringRef, 3> Redirects;
   SmallVector<const StringRef*, 3> RedirectPtrs;
@@ -112,7 +123,7 @@ bool DataAggregator::launchPerfEventsNoWait(StringRef PerfDataFilename) {
   return true;
 }
 
-bool DataAggregator::launchPerfTasksNoWait(StringRef PerfDataFilename) {
+bool DataAggregator::launchPerfTasksNoWait() {
   SmallVector<const char*, 4> Argv;
   SmallVector<StringRef, 3> Redirects;
   SmallVector<const StringRef*, 3> RedirectPtrs;
@@ -156,6 +167,88 @@ bool DataAggregator::launchPerfTasksNoWait(StringRef PerfDataFilename) {
   return true;
 }
 
+Optional<std::string> DataAggregator::getPerfBuildID() {
+  SmallVector<const char *, 4> Argv;
+  SmallVector<StringRef, 3> Redirects;
+  SmallVector<const StringRef*, 3> RedirectPtrs;
+  SmallVector<char, 256> OutputPath;
+  SmallVector<char, 256> ErrPath;
+
+  Argv.push_back(PerfPath.data());
+  Argv.push_back("buildid-list");
+  Argv.push_back("-i");
+  Argv.push_back(PerfDataFilename.data());
+  Argv.push_back(nullptr);
+
+  if (auto Errc = sys::fs::createTemporaryFile("perf.buildid", "out",
+                                               OutputPath)) {
+    outs() << "PERF2BOLT: Failed to create temporary file "
+           << OutputPath << " with error " << Errc.message() << "\n";
+    exit(1);
+  }
+
+  if (auto Errc = sys::fs::createTemporaryFile("perf.script", "err",
+                                               ErrPath)) {
+    outs() << "PERF2BOLT: Failed to create temporary file "
+           << ErrPath << " with error " << Errc.message() << "\n";
+    exit(1);
+  }
+
+  Redirects.push_back("");                           // Stdin
+  Redirects.push_back(StringRef(OutputPath.data())); // Stdout
+  Redirects.push_back(StringRef(ErrPath.data()));    // Stderr
+  RedirectPtrs.push_back(&Redirects[0]);
+  RedirectPtrs.push_back(&Redirects[1]);
+  RedirectPtrs.push_back(&Redirects[2]);
+
+  DEBUG(dbgs() << "Launching perf: " << PerfPath.data() << " 1> "
+               << OutputPath.data() << " 2> "
+               << ErrPath.data() << "\n");
+
+  auto RetCode = sys::ExecuteAndWait(PerfPath.data(), Argv.data(),
+                                     /*envp*/ nullptr, &RedirectPtrs[0]);
+
+  if (RetCode != 0) {
+    ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
+      MemoryBuffer::getFileOrSTDIN(ErrPath.data());
+    StringRef ErrBuf = (*MB)->getBuffer();
+
+    errs() << "PERF-ERROR: Return code " << RetCode << "\n";
+    errs() << ErrBuf;
+    deleteTempFile(ErrPath.data());
+    deleteTempFile(OutputPath.data());
+    return NoneType();
+  }
+
+  ErrorOr<std::unique_ptr<MemoryBuffer>> MB =
+    MemoryBuffer::getFileOrSTDIN(OutputPath.data());
+  if (std::error_code EC = MB.getError()) {
+    errs() << "Cannot open " << PerfTasksOutputPath.data() << ": "
+           << EC.message() << "\n";
+    deleteTempFile(ErrPath.data());
+    deleteTempFile(OutputPath.data());
+    return NoneType();
+  }
+
+  FileBuf.reset(MB->release());
+  ParsingBuf = FileBuf->getBuffer();
+  Col = 0;
+  Line = 1;
+  auto ParseResult = parsePerfBuildID();
+  if (!ParseResult) {
+    outs() << "PERF2BOLT: Failed to parse build-id from perf output\n";
+    deleteTempFile(ErrPath.data());
+    deleteTempFile(OutputPath.data());
+    return NoneType();
+  }
+
+  outs() << "PERF2BOLT: Perf.data build-id is:  " << *ParseResult << "\n";
+
+  deleteTempFile(ErrPath.data());
+  deleteTempFile(OutputPath.data());
+  return std::string(ParseResult->data(), ParseResult->size());
+}
+
 bool DataAggregator::checkPerfDataMagic(StringRef FileName) {
   int FD;
   if (sys::fs::openFileForRead(FileName, FD)) {
@@ -175,26 +268,18 @@ bool DataAggregator::checkPerfDataMagic(StringRef FileName) {
   return false;
 }
 
-void DataAggregator::deleteTempFiles() {
-  if (auto Errc = sys::fs::remove(PerfEventsErrPath.data())) {
+void DataAggregator::deleteTempFile(StringRef File) {
+  if (auto Errc = sys::fs::remove(File.data())) {
     outs() << "PERF2BOLT: Failed to delete temporary file "
-           << PerfEventsErrPath << " with error " << Errc.message() << "\n";
-  }
-
-  if (auto Errc = sys::fs::remove(PerfEventsOutputPath.data())) {
-    outs() << "PERF2BOLT: Failed to delete temporary file "
-           << PerfEventsOutputPath << " with error " << Errc.message() << "\n";
-  }
-
-  if (auto Errc = sys::fs::remove(PerfTasksErrPath.data())) {
-    outs() << "PERF2BOLT: Failed to delete temporary file "
-           << PerfTasksErrPath << " with error " << Errc.message() << "\n";
+           << File << " with error " << Errc.message() << "\n";
   }
+}
 
-  if (auto Errc = sys::fs::remove(PerfTasksOutputPath.data())) {
-    outs() << "PERF2BOLT: Failed to delete temporary file "
-           << PerfTasksOutputPath << " with error " << Errc.message() << "\n";
-  }
+void DataAggregator::deleteTempFiles() {
+  deleteTempFile(PerfEventsErrPath.data());
+  deleteTempFile(PerfEventsOutputPath.data());
+  deleteTempFile(PerfTasksErrPath.data());
+  deleteTempFile(PerfTasksOutputPath.data());
 }
 
 bool DataAggregator::aggregate(BinaryContext &BC,
@@ -541,7 +626,8 @@ std::error_code DataAggregator::parseEvents() {
       }
     }
     outs() << format("%.1f%%", Perc);
-    outs().resetColor();
+    if (outs().has_colors())
+      outs().resetColor();
     outs() << ")";
   }
   outs() << "\n";
@@ -623,6 +709,36 @@ std::error_code DataAggregator::parseTasks() {
   return std::error_code();
 }
 
+Optional<std::pair<StringRef, StringRef>>
+DataAggregator::parseNameBuildIDPair() {
+  while (checkAndConsumeFS()) {}
+
+  auto BuildIDStr = parseString(FieldSeparator, true);
+  if (std::error_code EC = BuildIDStr.getError())
+    return NoneType();
+
+  auto NameStr = parseString(FieldSeparator, true);
+  if (std::error_code EC = NameStr.getError())
+    return NoneType();
+
+  consumeRestOfLine();
+  return std::make_pair(NameStr.get(), BuildIDStr.get());
+}
+
+Optional<StringRef> DataAggregator::parsePerfBuildID() {
+  while (hasData()) {
+    auto IDPair = parseNameBuildIDPair();
+    if (!IDPair)
+      return NoneType();
+
+    if (sys::path::filename(IDPair->first) != BinaryName)
+      continue;
+
+    return IDPair->second;
+  }
+  return NoneType();
+}
+
 std::error_code DataAggregator::writeAggregatedFile() const {
   std::error_code EC;
   raw_fd_ostream OutFile(OutputFDataName, EC, sys::fs::OpenFlags::F_None);

bolt/DataAggregator.h

@@ -71,6 +71,9 @@ class DataAggregator : public DataReader {
   /// Whether aggregator was scheduled to run
   bool Enabled{false};
 
+  /// Input perf.data file
+  StringRef PerfDataFilename;
+
   /// Output file name to write aggregated fdata to
   StringRef OutputFDataName;
 
@@ -92,16 +95,17 @@ class DataAggregator : public DataReader {
 
   /// Launch a subprocess to read all perf samples and write them to an output
   /// file we will parse later
-  bool launchPerfEventsNoWait(StringRef PerfDataFilename);
+  bool launchPerfEventsNoWait();
 
   /// Launch a subprocess to read all perf task events. They contain the mapping
   /// of binary file name to PIDs used during data collection time. We later use
   /// the PIDs to filter samples.
-  bool launchPerfTasksNoWait(StringRef PerfDataFilename);
+  bool launchPerfTasksNoWait();
 
   /// Delete all temporary files created to hold the output generated by spawned
   /// subprocesses during the aggregation job
   void deleteTempFiles();
+  void deleteTempFile(StringRef File);
 
   // Semantic pass helpers
   /// Look up which function contains an address by using out map of
@@ -159,6 +163,13 @@ class DataAggregator : public DataReader {
   /// events with the association of binary file names and their PIDs.
   std::error_code parseTasks();
 
+  /// Parse a single pair of binary full path and associated build-id
+  Optional<std::pair<StringRef, StringRef>> parseNameBuildIDPair();
+
+  /// Parse the output generated by perf buildid-list to extract the build-id
+  /// of the binary used when collecting profiling
+  Optional<StringRef> parsePerfBuildID();
+
 public:
   DataAggregator(raw_ostream &Diag, StringRef BinaryName)
       : DataReader(Diag), BinaryName(llvm::sys::path::filename(BinaryName)) {}
@@ -174,6 +185,9 @@ class DataAggregator : public DataReader {
   /// job is in progress
   bool started() const { return Enabled; }
 
+  /// Force all subprocesses to stop and cancel aggregation
+  void abort();
+
   /// Dump data structures into a file readable by llvm-bolt
   std::error_code writeAggregatedFile() const;
 
@@ -184,13 +198,17 @@ class DataAggregator : public DataReader {
   /// Check whether \p FileName is a perf.data file
   static bool checkPerfDataMagic(StringRef FileName);
 
+  /// Launch a subprocess with perf buildid-list to extract the build-id of the
+  /// binary used when collecting profiling. Different than launchPerf*, this
+  /// one spawns the subprocess and blocks. Then it parses the result and
+  /// returns the build-id.
+  Optional<std::string> getPerfBuildID();
+
   /// Debugging dump methods
   void dump() const;
   void dump(const LBREntry &LBR) const;
   void dump(const PerfSample &Sample) const;
 };
-
-
 }
 }
 

bolt/RewriteInstance.cpp

@@ -325,6 +325,12 @@ AggregateOnly("aggregate-only",
   cl::Hidden,
   cl::cat(AggregatorCategory));
 
+static cl::opt<bool>
+IgnoreBuildID("ignore-build-id",
+  cl::desc("continue even if build-ids in input binary and perf.data mismatch"),
+  cl::init(false),
+  cl::cat(AggregatorCategory));
+
 // Check against lists of functions from options if we should
 // optimize the function with a given name.
 bool shouldProcess(const BinaryFunction &Function) {
@@ -774,6 +780,86 @@ void RewriteInstance::discoverStorage() {
   NewTextSegmentOffset = NextAvailableOffset;
 }
 
+Optional<std::string>
+RewriteInstance::getBuildID() {
+  for (auto &Section : InputFile->sections()) {
+    StringRef SectionName;
+    Section.getName(SectionName);
+
+    if (SectionName != ".note.gnu.build-id")
+      continue;
+
+    StringRef SectionContents;
+    Section.getContents(SectionContents);
+
+    // Reading notes section (see Portable Formats Specification, Version 1.1,
+    // pg 2-5, section "Note Section").
+    DataExtractor DE = DataExtractor(SectionContents, true, 8);
+    uint32_t Offset = 0;
+    if (!DE.isValidOffset(Offset))
+      return NoneType();
+    uint32_t NameSz = DE.getU32(&Offset);
+    if (!DE.isValidOffset(Offset))
+      return NoneType();
+    uint32_t DescSz = DE.getU32(&Offset);
+    if (!DE.isValidOffset(Offset))
+      return NoneType();
+    uint32_t Type = DE.getU32(&Offset);
+
+    DEBUG(dbgs() << "NameSz = " << NameSz << "; DescSz = " << DescSz
+                 << "; Type = " << Type << "\n");
+
+    // Type 3 is a GNU build-id note section
+    if (Type != 3)
+      return NoneType();
+
+    StringRef Name = SectionContents.slice(Offset, Offset + NameSz);
+    Offset = RoundUpToAlignment(Offset + NameSz, 4);
+    StringRef BinaryBuildID = SectionContents.slice(Offset, Offset + DescSz);
+    if (Name.substr(0, 3) != "GNU")
+      return NoneType();
+
+    std::string Str;
+    raw_string_ostream OS(Str);
+    auto CharIter = BinaryBuildID.bytes_begin();
+    while (CharIter != BinaryBuildID.bytes_end()) {
+      if (*CharIter < 0x10)
+        OS << "0";
+      OS << Twine::utohexstr(*CharIter);
+      ++CharIter;
+    }
+    outs() << "BOLT-INFO: Binary build-id is:     " << OS.str() << "\n";
+    return OS.str();
+  }
+  return NoneType();
+}
+
+void RewriteInstance::checkBuildID() {
+  auto FileBuildID = getBuildID();
+  if (!FileBuildID) {
+    outs() << "BOLT-WARNING: Build ID will not be checked because we could not "
+              "read one from input binary\n";
+    return;
+  }
+  auto PerfBuildID = DA.getPerfBuildID();
+  if (!PerfBuildID) {
+    outs() << "BOLT-WARNING: Build ID will not be checked because we could not "
+              "read one from perf.data\n";
+    return;
+  }
+  if (*FileBuildID == *PerfBuildID)
+    return;
+
+  outs() << "BOLT-ERROR: Build ID mismatch! This indicates the input binary "
+            "supplied for data aggregation is not the same recorded by perf "
+            "when collecting profiling data.\n";
+
+  if (!opts::IgnoreBuildID) {
+    DA.abort();
+    exit(1);
+  }
+}
+
 void RewriteInstance::run() {
   if (!BC) {
     errs() << "BOLT-ERROR: failed to create a binary context\n";
@@ -806,6 +892,8 @@ void RewriteInstance::run() {
                 (llvm::Triple::ArchType)InputFile->getArch())
          << "\n";
 
+  if (DA.started())
+    checkBuildID();
   unsigned PassNumber = 1;
   executeRewritePass({});
   if (opts::AggregateOnly)