Reland "[lld-macho] Show source information for undefined references"

The error used to look like this:

  ld64.lld: error: undefined symbol: _foo
  >>> referenced by /path/to/bar.o:(symbol _baz+0x4)

If DWARF line information is available, we now show where in the source
the references are coming from:

  ld64.lld: error: unreferenced symbol: _foo
  >>> referenced by: bar.cpp:42 (/path/to/bar.cpp:42)
  >>>                /path/to/bar.o:(symbol _baz+0x4)

The reland is identical to the first time this landed. The fix was in D128294.
This reverts commit 0cc7ad4.

Differential Revision: https://reviews.llvm.org/D128184

Author: BertalanD

lld/ELF/InputFiles.cpp

@@ -224,11 +224,11 @@ template <class ELFT>
 static std::string getSrcMsgAux(ObjFile<ELFT> &file, const Symbol &sym,
                                 InputSectionBase &sec, uint64_t offset) {
   // In DWARF, functions and variables are stored to different places.
-  // First, lookup a function for a given offset.
+  // First, look up a function for a given offset.
   if (Optional<DILineInfo> info = file.getDILineInfo(&sec, offset))
     return createFileLineMsg(info->FileName, info->Line);
 
-  // If it failed, lookup again as a variable.
+  // If it failed, look up again as a variable.
   if (Optional<std::pair<std::string, unsigned>> fileLine =
           file.getVariableLoc(sym.getName()))
     return createFileLineMsg(fileLine->first, fileLine->second);

lld/MachO/Dwarf.cpp

@@ -20,15 +20,16 @@ using namespace llvm;
 std::unique_ptr<DwarfObject> DwarfObject::create(ObjFile *obj) {
   auto dObj = std::make_unique<DwarfObject>();
   bool hasDwarfInfo = false;
-  // LLD only needs to extract the source file path from the debug info, so we
-  // initialize DwarfObject with just the sections necessary to get that path.
-  // The debugger will locate the debug info via the object file paths that we
-  // emit in our STABS symbols, so we don't need to process & emit them
-  // ourselves.
+  // LLD only needs to extract the source file path and line numbers from the
+  // debug info, so we initialize DwarfObject with just the sections necessary
+  // to get that path. The debugger will locate the debug info via the object
+  // file paths that we emit in our STABS symbols, so we don't need to process &
+  // emit them ourselves.
   for (const InputSection *isec : obj->debugSections) {
     if (StringRef *s =
             StringSwitch<StringRef *>(isec->getName())
                 .Case(section_names::debugInfo, &dObj->infoSection.Data)
+                .Case(section_names::debugLine, &dObj->lineSection.Data)
                 .Case(section_names::debugAbbrev, &dObj->abbrevSection)
                 .Case(section_names::debugStr, &dObj->strSection)
                 .Default(nullptr)) {

lld/MachO/Dwarf.h

@@ -37,12 +37,17 @@ class DwarfObject final : public llvm::DWARFObject {
   llvm::StringRef getAbbrevSection() const override { return abbrevSection; }
   llvm::StringRef getStrSection() const override { return strSection; }
 
+  llvm::DWARFSection const &getLineSection() const override {
+    return lineSection;
+  }
+
   // Returns an instance of DwarfObject if the given object file has the
   // relevant DWARF debug sections.
   static std::unique_ptr<DwarfObject> create(ObjFile *);
 
 private:
   llvm::DWARFSection infoSection;
+  llvm::DWARFSection lineSection;
   llvm::StringRef abbrevSection;
   llvm::StringRef strSection;
 };

lld/MachO/InputFiles.cpp

@@ -998,6 +998,8 @@ void ObjFile::parseDebugInfo() {
   if (!dObj)
     return;
 
+  // We do not re-use the context from getDwarf() here as that function
+  // constructs an expensive DWARFCache object.
   auto *ctx = make<DWARFContext>(
       std::move(dObj), "",
       [&](Error err) {
@@ -1373,6 +1375,31 @@ void ObjFile::registerEhFrames(Section &ehFrameSection) {
   }
 }
 
+std::string ObjFile::sourceFile() const {
+  SmallString<261> dir(compileUnit->getCompilationDir());
+  StringRef sep = sys::path::get_separator();
+  // We don't use `path::append` here because we want an empty `dir` to result
+  // in an absolute path. `append` would give us a relative path for that case.
+  if (!dir.endswith(sep))
+    dir += sep;
+  return (dir + compileUnit->getUnitDIE().getShortName()).str();
+}
+
+lld::DWARFCache *ObjFile::getDwarf() {
+  llvm::call_once(initDwarf, [this]() {
+    auto dwObj = DwarfObject::create(this);
+    if (!dwObj)
+      return;
+    dwarfCache = std::make_unique<DWARFCache>(std::make_unique<DWARFContext>(
+        std::move(dwObj), "",
+        [&](Error err) { warn(getName() + ": " + toString(std::move(err))); },
+        [&](Error warning) {
+          warn(getName() + ": " + toString(std::move(warning)));
+        }));
+  });
+
+  return dwarfCache.get();
+}
 // The path can point to either a dylib or a .tbd file.
 static DylibFile *loadDylib(StringRef path, DylibFile *umbrella) {
   Optional<MemoryBufferRef> mbref = readFile(path);

lld/MachO/InputFiles.h

@@ -12,6 +12,7 @@
 #include "MachOStructs.h"
 #include "Target.h"
 
+#include "lld/Common/DWARF.h"
 #include "lld/Common/LLVM.h"
 #include "lld/Common/Memory.h"
 #include "llvm/ADT/CachedHashString.h"
@@ -21,6 +22,7 @@
 #include "llvm/DebugInfo/DWARF/DWARFUnit.h"
 #include "llvm/Object/Archive.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Threading.h"
 #include "llvm/TextAPI/TextAPIReader.h"
 
 #include <vector>
@@ -159,14 +161,21 @@ class ObjFile final : public InputFile {
 
   static bool classof(const InputFile *f) { return f->kind() == ObjKind; }
 
+  std::string sourceFile() const;
+  // Parses line table information for diagnostics. compileUnit should be used
+  // for other purposes.
+  lld::DWARFCache *getDwarf();
+
   llvm::DWARFUnit *compileUnit = nullptr;
+  std::unique_ptr<lld::DWARFCache> dwarfCache;
   Section *addrSigSection = nullptr;
   const uint32_t modTime;
   std::vector<ConcatInputSection *> debugSections;
   std::vector<CallGraphEntry> callGraph;
   llvm::DenseMap<ConcatInputSection *, FDE> fdes;
 
 private:
+  llvm::once_flag initDwarf;
   template <class LP> void parseLazy();
   template <class SectionHeader> void parseSections(ArrayRef<SectionHeader>);
   template <class LP>

lld/MachO/InputSection.cpp

@@ -55,17 +55,21 @@ static uint64_t resolveSymbolVA(const Symbol *sym, uint8_t type) {
   return sym->getVA();
 }
 
+const Defined *InputSection::getContainingSymbol(uint64_t off) const {
+  auto *nextSym = llvm::upper_bound(
+      symbols, off, [](uint64_t a, const Defined *b) { return a < b->value; });
+  if (nextSym == symbols.begin())
+    return nullptr;
+  return *std::prev(nextSym);
+}
+
 std::string InputSection::getLocation(uint64_t off) const {
   // First, try to find a symbol that's near the offset. Use it as a reference
   // point.
-  auto *nextSym = llvm::upper_bound(
-      symbols, off, [](uint64_t a, const Defined *b) { return a < b->value; });
-  if (nextSym != symbols.begin()) {
-    auto &sym = *std::prev(nextSym);
+  if (auto *sym = getContainingSymbol(off))
     return (toString(getFile()) + ":(symbol " + sym->getName() + "+0x" +
             Twine::utohexstr(off - sym->value) + ")")
         .str();
-  }
 
   // If that fails, use the section itself as a reference point.
   for (const Subsection &subsec : section.subsections) {
@@ -74,11 +78,61 @@ std::string InputSection::getLocation(uint64_t off) const {
       break;
     }
   }
+
   return (toString(getFile()) + ":(" + getName() + "+0x" +
           Twine::utohexstr(off) + ")")
       .str();
 }
 
+std::string InputSection::getSourceLocation(uint64_t off) const {
+  auto *obj = dyn_cast<ObjFile>(getFile());
+  if (!obj)
+    return {};
+
+  DWARFCache *dwarf = obj->getDwarf();
+  if (!dwarf)
+    return std::string();
+
+  for (const Subsection &subsec : section.subsections) {
+    if (subsec.isec == this) {
+      off += subsec.offset;
+      break;
+    }
+  }
+
+  auto createMsg = [&](StringRef path, unsigned line) {
+    std::string filename = sys::path::filename(path).str();
+    std::string lineStr = (":" + Twine(line)).str();
+    if (filename == path)
+      return filename + lineStr;
+    return (filename + lineStr + " (" + path + lineStr + ")").str();
+  };
+
+  // First, look up a function for a given offset.
+  if (Optional<DILineInfo> li = dwarf->getDILineInfo(
+          section.addr + off, object::SectionedAddress::UndefSection))
+    return createMsg(li->FileName, li->Line);
+
+  // If it failed, look up again as a variable.
+  if (const Defined *sym = getContainingSymbol(off)) {
+    // Symbols are generally prefixed with an underscore, which is not included
+    // in the debug information.
+    StringRef symName = sym->getName();
+    if (!symName.empty() && symName[0] == '_')
+      symName = symName.substr(1);
+
+    if (Optional<std::pair<std::string, unsigned>> fileLine =
+            dwarf->getVariableLoc(symName))
+      return createMsg(fileLine->first, fileLine->second);
+  }
+
+  // Try to get the source file's name from the DWARF information.
+  if (obj->compileUnit)
+    return obj->sourceFile();
+
+  return {};
+}
+
 void ConcatInputSection::foldIdentical(ConcatInputSection *copy) {
   align = std::max(align, copy->align);
   copy->live = false;

lld/MachO/InputSection.h

@@ -50,7 +50,11 @@ class InputSection {
   // The offset from the beginning of the file.
   uint64_t getVA(uint64_t off) const;
   // Return a user-friendly string for use in diagnostics.
+  // Format: /path/to/object.o:(symbol _func+0x123)
   std::string getLocation(uint64_t off) const;
+  // Return the source line corresponding to an address, or the empty string.
+  // Format: Source.cpp:123 (/path/to/Source.cpp:123)
+  std::string getSourceLocation(uint64_t off) const;
   // Whether the data at \p off in this InputSection is live.
   virtual bool isLive(uint64_t off) const = 0;
   virtual void markLive(uint64_t off) = 0;
@@ -85,6 +89,8 @@ class InputSection {
 
 protected:
   const Section &section;
+
+  const Defined *getContainingSymbol(uint64_t off) const;
 };
 
 // ConcatInputSections are combined into (Concat)OutputSections through simple
@@ -292,6 +298,7 @@ constexpr const char compactUnwind[] = "__compact_unwind";
 constexpr const char data[] = "__data";
 constexpr const char debugAbbrev[] = "__debug_abbrev";
 constexpr const char debugInfo[] = "__debug_info";
+constexpr const char debugLine[] = "__debug_line";
 constexpr const char debugStr[] = "__debug_str";
 constexpr const char ehFrame[] = "__eh_frame";
 constexpr const char gccExceptTab[] = "__gcc_except_tab";

lld/MachO/SymbolTable.cpp

@@ -381,8 +381,11 @@ void macho::reportPendingUndefinedSymbols() {
          locations.codeReferences) {
       if (i >= maxUndefinedReferences)
         break;
-      // TODO: Get source file/line from debug information.
-      message += "\n>>> referenced by " + loc.isec->getLocation(loc.offset);
+      message += "\n>>> referenced by ";
+      std::string src = loc.isec->getSourceLocation(loc.offset);
+      if (!src.empty())
+        message += src + "\n>>>               ";
+      message += loc.isec->getLocation(loc.offset);
       ++i;
     }
 

lld/MachO/SyntheticSections.cpp

@@ -852,16 +852,9 @@ SymtabSection::SymtabSection(StringTableSection &stringTableSection)
     : LinkEditSection(segment_names::linkEdit, section_names::symbolTable),
       stringTableSection(stringTableSection) {}
 
-void SymtabSection::emitBeginSourceStab(DWARFUnit *compileUnit) {
+void SymtabSection::emitBeginSourceStab(StringRef sourceFile) {
   StabsEntry stab(N_SO);
-  SmallString<261> dir(compileUnit->getCompilationDir());
-  StringRef sep = sys::path::get_separator();
-  // We don't use `path::append` here because we want an empty `dir` to result
-  // in an absolute path. `append` would give us a relative path for that case.
-  if (!dir.endswith(sep))
-    dir += sep;
-  stab.strx = stringTableSection.addString(
-      saver().save(dir + compileUnit->getUnitDIE().getShortName()));
+  stab.strx = stringTableSection.addString(saver().save(sourceFile));
   stabs.emplace_back(std::move(stab));
 }
 
@@ -956,7 +949,7 @@ void SymtabSection::emitStabs() {
         emitEndSourceStab();
       lastFile = file;
 
-      emitBeginSourceStab(file->compileUnit);
+      emitBeginSourceStab(file->sourceFile());
       emitObjectFileStab(file);
     }
 

lld/MachO/SyntheticSections.h

@@ -435,7 +435,7 @@ class SymtabSection : public LinkEditSection {
   uint32_t getNumUndefinedSymbols() const { return undefinedSymbols.size(); }
 
 private:
-  void emitBeginSourceStab(llvm::DWARFUnit *compileUnit);
+  void emitBeginSourceStab(StringRef);
   void emitEndSourceStab();
   void emitObjectFileStab(ObjFile *);
   void emitEndFunStab(Defined *);