[InstrProf] Add debuginfod correlation support (#106606)

This patch adds debuginfod support into llvm-profdata to
find the assosicated executable by a build id in a raw
profile to correlate a profile with a provided correlation
kind (debug-info or binary).

Author: gulfemsavrun

compiler-rt/test/profile/Linux/instrprof-debug-info-correlate-debuginfod.c

@@ -0,0 +1,24 @@
+// REQUIRES: curl
+// RUN: rm -rf %t
+
+// Default instrumented build with no profile correlation.
+// RUN: %clang_pgogen -o %t.default -Wl,--build-id=0x12345678 -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp
+// RUN: env LLVM_PROFILE_FILE=%t.profraw %run %t.default
+// RUN: llvm-profdata merge -o %t.default.profdata %t.profraw
+
+// Build with profile debuginfo correlation.
+// RUN: %clang_pgogen -o %t.correlate.exe -Wl,--build-id=0x12345678 -g -gdwarf-4 -mllvm --debug-info-correlate -mllvm --disable-vp=true %S/../Inputs/instrprof-debug-info-correlate-main.cpp %S/../Inputs/instrprof-debug-info-correlate-foo.cpp
+// RUN: env LLVM_PROFILE_FILE=%t.debug-info-correlate.proflite %run %t.correlate.exe
+
+// Test llvm-profdata merge profile correlation with --debuginfod option.
+// RUN: mkdir -p %t/buildid/12345678
+// RUN: cp %t.correlate.exe %t/buildid/12345678/debuginfo
+// RUN: mkdir -p %t/debuginfod-cache
+// RUN: env DEBUGINFOD_CACHE_PATH=%t/debuginfod-cache DEBUGINFOD_URLS=file://%t llvm-profdata merge -o %t.correlate-debuginfod.profdata --debuginfod --correlate=debug-info %t.debug-info-correlate.proflite
+// RUN: diff <(llvm-profdata show --all-functions --counts %t.default.profdata) <(llvm-profdata show --all-functions --counts %t.correlate-debuginfod.profdata)
+
+// Test llvm-profdata merge profile correlation with --debug-file-directory option.
+// RUN: mkdir -p %t/.build-id/12
+// RUN: cp %t.correlate.exe %t/.build-id/12/345678.debug
+// RUN: llvm-profdata merge -o %t.correlate-debug-file-dir.profdata --debug-file-directory %t --correlate=debug-info %t.debug-info-correlate.proflite
+// RUN: diff <(llvm-profdata show --all-functions --counts %t.default.profdata) <(llvm-profdata show --all-functions --counts %t.correlate-debug-file-dir.profdata)

compiler-rt/test/profile/instrprof-binary-correlate-debuginfod.c

@@ -0,0 +1,41 @@
+// REQUIRES: linux || windows
+// REQUIRES: curl
+
+// Default instrumented build with no profile correlation.
+// RUN: %clang_profgen -o %t.default.exe -Wl,--build-id=0x12345678 -fprofile-instr-generate -fcoverage-mapping %S/Inputs/instrprof-debug-info-correlate-main.cpp %S/Inputs/instrprof-debug-info-correlate-foo.cpp
+// RUN: env LLVM_PROFILE_FILE=%t.default.profraw %run %t.default.exe
+// RUN: llvm-profdata merge -o %t.default.profdata %t.default.profraw
+// RUN: llvm-profdata show --all-functions --counts %t.default.profdata > %t.default.profdata.show
+
+// Build with profile binary correlation.
+// RUN: %clang_profgen -o %t.correlate.exe -Wl,--build-id=0x12345678 -fprofile-instr-generate -fcoverage-mapping -mllvm -profile-correlate=binary %S/Inputs/instrprof-debug-info-correlate-main.cpp %S/Inputs/instrprof-debug-info-correlate-foo.cpp
+// Strip above binary and run
+// RUN: llvm-strip %t.correlate.exe -o %t.stripped.exe
+// RUN: env LLVM_PROFILE_FILE=%t.correlate.profraw %run %t.stripped.exe
+
+// Test llvm-profdata merge profile correlation with --debuginfod option.
+// RUN: mkdir -p %t/buildid/12345678
+// RUN: cp %t.correlate.exe %t/buildid/12345678/debuginfo
+// RUN: mkdir -p %t/debuginfod-cache
+// RUN: env DEBUGINFOD_CACHE_PATH=%t/debuginfod-cache DEBUGINFOD_URLS=file://%t llvm-profdata merge -o %t.correlate-debuginfod.profdata --debuginfod --correlate=binary %t.correlate.profraw
+// RUN: llvm-profdata show --all-functions --counts %t.correlate-debuginfod.profdata > %t.correlate-debuginfod.profdata.show
+// RUN: diff %t.default.profdata.show %t.correlate-debuginfod.profdata.show
+
+// Test llvm-profdata merge profile correlation with --debug-file-directory option.
+// RUN: mkdir -p %t/.build-id/12
+// RUN: cp %t.correlate.exe %t/.build-id/12/345678.debug
+// RUN: llvm-profdata merge -o %t.correlate-debug-file-dir.profdata --debug-file-directory %t --correlate=binary %t.correlate.profraw
+// RUN: llvm-profdata show --all-functions --counts %t.correlate-debug-file-dir.profdata > %t.correlate-debug-file-dir.profdata.show
+// RUN: diff %t.default.profdata.show %t.correlate-debug-file-dir.profdata.show
+
+// Test error for llvm-profdata merge profile correlation with only --correlate=binary option.
+// RUN: not llvm-profdata merge -o %t.correlate-error.profdata --correlate=binary %t.correlate.profraw 2>&1 | FileCheck %s --check-prefix=MISSING-FLAG
+// MISSING-FLAG: error: Expected --debuginfod or --debug-file-directory when --correlate is provided
+
+// Test error for llvm-profdata merge profile correlation when a proper --correlate option is not provided.
+// RUN: not llvm-profdata merge -o %t.correlate-error.profdata --debug-file-directory %t --correlate="" %t.correlate.profraw 2>&1 | FileCheck %s --check-prefix=MISSING-CORRELATION-KIND
+// MISSING-CORRELATION-KIND: error: Expected --correlate when --debug-file-directory is provided
+
+// Test error for llvm-profdata merge profile correlation with mixing correlation options.
+// RUN: not llvm-profdata merge -o %t.error.profdata --binary-file=%t.correlate.exe --debug-file-directory %t --correlate=binary %t.correlate.profraw  2>&1 | FileCheck %s --check-prefix=MIXING-FLAGS
+// MIXING-FLAGS: error: Expected only one of -binary-file, -debuginfod or -debug-file-directory

compiler-rt/test/profile/lit.cfg.py

@@ -179,3 +179,6 @@ def exclude_unsupported_files_for_aix(dirname):
 
 if config.android:
     config.unsupported = True
+
+if config.have_curl:
+    config.available_features.add("curl")

llvm/docs/CommandGuide/llvm-profdata.rst

@@ -210,6 +210,25 @@ OPTIONS
  the raw profile. When ``-profile-correlate=binary`` was used for
  instrumentation, use this option to correlate the raw profile.
 
+.. option:: --debuginfod
+
+ Use debuginfod to find the associated executables that contain profile data and
+ name sections for the raw profiles to correlate them.
+ When -profile-correlate=binary was used for instrumentation, this option can be
+ used for correlation.
+
+.. option:: --debug-file-directory=<dir>
+
+ Use provided local directories to search for executables that contain profile
+ data and name sections for the raw profiles to correlate them.
+ When -profile-correlate=binary was used for instrumentation, this option can be
+ used for correlation.
+
+.. option:: --correlate=<kind>
+
+ Specify the correlation kind (debug_info or binary) to use when -debuginfod or
+ -debug-file-directory=<dir> option is provided.
+
 .. option:: --temporal-profile-trace-reservoir-size
 
  The maximum number of temporal profile traces to be stored in the output

llvm/include/llvm/ProfileData/InstrProfCorrelator.h

@@ -13,6 +13,8 @@
 #define LLVM_PROFILEDATA_INSTRPROFCORRELATOR_H
 
 #include "llvm/ADT/DenseSet.h"
+#include "llvm/Debuginfod/BuildIDFetcher.h"
+#include "llvm/Object/BuildID.h"
 #include "llvm/ProfileData/InstrProf.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -36,7 +38,9 @@ class InstrProfCorrelator {
   enum ProfCorrelatorKind { NONE, DEBUG_INFO, BINARY };
 
   static llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
-  get(StringRef Filename, ProfCorrelatorKind FileKind);
+  get(StringRef Filename, ProfCorrelatorKind FileKind,
+      const object::BuildIDFetcher *BIDFetcher = nullptr,
+      const ArrayRef<llvm::object::BuildID> BIs = std::nullopt);
 
   /// Construct a ProfileData vector used to correlate raw instrumentation data
   /// to their functions.

llvm/include/llvm/ProfileData/InstrProfReader.h

@@ -197,15 +197,21 @@ class InstrProfReader {
 
   /// Factory method to create an appropriately typed reader for the given
   /// instrprof file.
-  static Expected<std::unique_ptr<InstrProfReader>>
-  create(const Twine &Path, vfs::FileSystem &FS,
-         const InstrProfCorrelator *Correlator = nullptr,
-         std::function<void(Error)> Warn = nullptr);
-
-  static Expected<std::unique_ptr<InstrProfReader>>
-  create(std::unique_ptr<MemoryBuffer> Buffer,
-         const InstrProfCorrelator *Correlator = nullptr,
-         std::function<void(Error)> Warn = nullptr);
+  static Expected<std::unique_ptr<InstrProfReader>> create(
+      const Twine &Path, vfs::FileSystem &FS,
+      const InstrProfCorrelator *Correlator = nullptr,
+      const object::BuildIDFetcher *BIDFetcher = nullptr,
+      const InstrProfCorrelator::ProfCorrelatorKind BIDFetcherCorrelatorKind =
+          InstrProfCorrelator::ProfCorrelatorKind::NONE,
+      std::function<void(Error)> Warn = nullptr);
+
+  static Expected<std::unique_ptr<InstrProfReader>> create(
+      std::unique_ptr<MemoryBuffer> Buffer,
+      const InstrProfCorrelator *Correlator = nullptr,
+      const object::BuildIDFetcher *BIDFetcher = nullptr,
+      const InstrProfCorrelator::ProfCorrelatorKind BIDFetcherCorrelatorKind =
+          InstrProfCorrelator::ProfCorrelatorKind::NONE,
+      std::function<void(Error)> Warn = nullptr);
 
   /// \param Weight for raw profiles use this as the temporal profile trace
   ///               weight
@@ -314,6 +320,14 @@ class RawInstrProfReader : public InstrProfReader {
   /// If available, this hold the ProfileData array used to correlate raw
   /// instrumentation data to their functions.
   const InstrProfCorrelatorImpl<IntPtrT> *Correlator;
+  /// Fetches debuginfo by build id to correlate profiles.
+  const object::BuildIDFetcher *BIDFetcher;
+  /// Correlates profiles with build id fetcher by fetching debuginfo with build
+  /// ID.
+  std::unique_ptr<InstrProfCorrelator> BIDFetcherCorrelator;
+  /// Indicates if should use debuginfo or binary to correlate with build id
+  /// fetcher.
+  InstrProfCorrelator::ProfCorrelatorKind BIDFetcherCorrelatorKind;
   /// A list of timestamps paired with a function name reference.
   std::vector<std::pair<uint64_t, uint64_t>> TemporalProfTimestamps;
   bool ShouldSwapBytes;
@@ -349,13 +363,18 @@ class RawInstrProfReader : public InstrProfReader {
   static const uint64_t MaxCounterValue = (1ULL << 56);
 
 public:
-  RawInstrProfReader(std::unique_ptr<MemoryBuffer> DataBuffer,
-                     const InstrProfCorrelator *Correlator,
-                     std::function<void(Error)> Warn)
+  RawInstrProfReader(
+      std::unique_ptr<MemoryBuffer> DataBuffer,
+      const InstrProfCorrelator *Correlator,
+      const object::BuildIDFetcher *BIDFetcher,
+      const InstrProfCorrelator::ProfCorrelatorKind BIDFetcherCorrelatorKind,
+      std::function<void(Error)> Warn)
       : DataBuffer(std::move(DataBuffer)),
         Correlator(dyn_cast_or_null<const InstrProfCorrelatorImpl<IntPtrT>>(
             Correlator)),
-        Warn(Warn) {}
+        BIDFetcher(BIDFetcher),
+        BIDFetcherCorrelatorKind(BIDFetcherCorrelatorKind), Warn(Warn) {}
+
   RawInstrProfReader(const RawInstrProfReader &) = delete;
   RawInstrProfReader &operator=(const RawInstrProfReader &) = delete;
 
@@ -439,7 +458,7 @@ class RawInstrProfReader : public InstrProfReader {
 
   void advanceData() {
     // `CountersDelta` is a constant zero when using debug info correlation.
-    if (!Correlator) {
+    if (!Correlator && !BIDFetcherCorrelator) {
       // The initial CountersDelta is the in-memory address difference between
       // the data and counts sections:
       // start(__llvm_prf_cnts) - start(__llvm_prf_data)

llvm/lib/ProfileData/InstrProfCorrelator.cpp

@@ -91,7 +91,31 @@ InstrProfCorrelator::Context::get(std::unique_ptr<MemoryBuffer> Buffer,
 }
 
 llvm::Expected<std::unique_ptr<InstrProfCorrelator>>
-InstrProfCorrelator::get(StringRef Filename, ProfCorrelatorKind FileKind) {
+InstrProfCorrelator::get(StringRef Filename, ProfCorrelatorKind FileKind,
+                         const object::BuildIDFetcher *BIDFetcher,
+                         const ArrayRef<object::BuildID> BIs) {
+  std::optional<std::string> Path;
+  if (BIDFetcher) {
+    if (BIs.empty())
+      return make_error<InstrProfError>(
+          instrprof_error::unable_to_correlate_profile,
+          "unsupported profile binary correlation when there is no build ID "
+          "in a profile");
+    if (BIs.size() > 1)
+      return make_error<InstrProfError>(
+          instrprof_error::unable_to_correlate_profile,
+          "unsupported profile binary correlation when there are multiple "
+          "build IDs in a profile");
+
+    Path = BIDFetcher->fetch(BIs.front());
+    if (!Path)
+      return make_error<InstrProfError>(
+          instrprof_error::unable_to_correlate_profile,
+          "Missing build ID: " + llvm::toHex(BIs.front(),
+                                             /*LowerCase=*/true));
+    Filename = *Path;
+  }
+
   if (FileKind == DEBUG_INFO) {
     auto DsymObjectsOrErr =
         object::MachOObjectFile::findDsymObjectMembers(Filename);

llvm/lib/ProfileData/InstrProfReader.cpp

@@ -150,22 +150,25 @@ static void printBinaryIdsInternal(raw_ostream &OS,
   }
 }
 
-Expected<std::unique_ptr<InstrProfReader>>
-InstrProfReader::create(const Twine &Path, vfs::FileSystem &FS,
-                        const InstrProfCorrelator *Correlator,
-                        std::function<void(Error)> Warn) {
+Expected<std::unique_ptr<InstrProfReader>> InstrProfReader::create(
+    const Twine &Path, vfs::FileSystem &FS,
+    const InstrProfCorrelator *Correlator,
+    const object::BuildIDFetcher *BIDFetcher,
+    const InstrProfCorrelator::ProfCorrelatorKind BIDFetcherCorrelatorKind,
+    std::function<void(Error)> Warn) {
   // Set up the buffer to read.
   auto BufferOrError = setupMemoryBuffer(Path, FS);
   if (Error E = BufferOrError.takeError())
     return std::move(E);
   return InstrProfReader::create(std::move(BufferOrError.get()), Correlator,
-                                 Warn);
+                                 BIDFetcher, BIDFetcherCorrelatorKind, Warn);
 }
 
-Expected<std::unique_ptr<InstrProfReader>>
-InstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer,
-                        const InstrProfCorrelator *Correlator,
-                        std::function<void(Error)> Warn) {
+Expected<std::unique_ptr<InstrProfReader>> InstrProfReader::create(
+    std::unique_ptr<MemoryBuffer> Buffer, const InstrProfCorrelator *Correlator,
+    const object::BuildIDFetcher *BIDFetcher,
+    const InstrProfCorrelator::ProfCorrelatorKind BIDFetcherCorrelatorKind,
+    std::function<void(Error)> Warn) {
   if (Buffer->getBufferSize() == 0)
     return make_error<InstrProfError>(instrprof_error::empty_raw_profile);
 
@@ -174,9 +177,13 @@ InstrProfReader::create(std::unique_ptr<MemoryBuffer> Buffer,
   if (IndexedInstrProfReader::hasFormat(*Buffer))
     Result.reset(new IndexedInstrProfReader(std::move(Buffer)));
   else if (RawInstrProfReader64::hasFormat(*Buffer))
-    Result.reset(new RawInstrProfReader64(std::move(Buffer), Correlator, Warn));
+    Result.reset(new RawInstrProfReader64(std::move(Buffer), Correlator,
+                                          BIDFetcher, BIDFetcherCorrelatorKind,
+                                          Warn));
   else if (RawInstrProfReader32::hasFormat(*Buffer))
-    Result.reset(new RawInstrProfReader32(std::move(Buffer), Correlator, Warn));
+    Result.reset(new RawInstrProfReader32(std::move(Buffer), Correlator,
+                                          BIDFetcher, BIDFetcherCorrelatorKind,
+                                          Warn));
   else if (TextInstrProfReader::hasFormat(*Buffer))
     Result.reset(new TextInstrProfReader(std::move(Buffer)));
   else
@@ -633,6 +640,19 @@ Error RawInstrProfReader<IntPtrT>::readHeader(
   if (Start + ValueDataOffset > DataBuffer->getBufferEnd())
     return error(instrprof_error::bad_header);
 
+  if (BIDFetcher) {
+    std::vector<object::BuildID> BinaryIDs;
+    if (Error E = readBinaryIds(BinaryIDs))
+      return E;
+    if (auto E = InstrProfCorrelator::get("", BIDFetcherCorrelatorKind,
+                                          BIDFetcher, BinaryIDs)
+                     .moveInto(BIDFetcherCorrelator)) {
+      return E;
+    }
+    if (auto Err = BIDFetcherCorrelator->correlateProfileData(0))
+      return Err;
+  }
+
   if (Correlator) {
     // These sizes in the raw file are zero because we constructed them in the
     // Correlator.
@@ -643,6 +663,14 @@ Error RawInstrProfReader<IntPtrT>::readHeader(
     DataEnd = Data + Correlator->getDataSize();
     NamesStart = Correlator->getNamesPointer();
     NamesEnd = NamesStart + Correlator->getNamesSize();
+  } else if (BIDFetcherCorrelator) {
+    InstrProfCorrelatorImpl<IntPtrT> *BIDFetcherCorrelatorImpl =
+        dyn_cast_or_null<InstrProfCorrelatorImpl<IntPtrT>>(
+            BIDFetcherCorrelator.get());
+    Data = BIDFetcherCorrelatorImpl->getDataPointer();
+    DataEnd = Data + BIDFetcherCorrelatorImpl->getDataSize();
+    NamesStart = BIDFetcherCorrelatorImpl->getNamesPointer();
+    NamesEnd = NamesStart + BIDFetcherCorrelatorImpl->getNamesSize();
   } else {
     Data = reinterpret_cast<const RawInstrProf::ProfileData<IntPtrT> *>(
         Start + DataOffset);

llvm/tools/llvm-profdata/CMakeLists.txt

@@ -12,3 +12,7 @@ add_llvm_tool(llvm-profdata
   intrinsics_gen
   GENERATE_DRIVER
   )
+
+if(NOT LLVM_TOOL_LLVM_DRIVER_BUILD)
+  target_link_libraries(llvm-profdata PRIVATE LLVMDebuginfod)
+endif()