[lldb] Introduce new SymbolFileJSON and ObjectFileJSON

Introduce a new object and symbol file format with the goal of mapping
addresses to symbol names. I'd like to think of is as an extremely
simple textual symtab. The file format consists of a triple, a UUID and
a list of symbols. JSON is used for the encoding, but that's mostly an
implementation detail. The goal of the format was to be simple and human
readable.

The new file format is motivated by two use cases:

 - Stripped binaries: when a binary is stripped, you lose the ability to
   do thing like setting symbolic breakpoints. You can keep the
   unstripped binary around, but if all you need is the stripped
   symbols then that's a lot of overhead. Instead, we could save the
   stripped symbols to a file and load them in the debugger when
   needed. I want to extend llvm-strip to have a mode where it emits
   this new file format.

 - Interactive crashlogs: with interactive crashlogs, if we don't have
   the binary or the dSYM for a particular module, we currently show an
   unnamed symbol for those frames. This is a regression compared to the
   textual format, that has these frames pre-symbolicated. Given that
   this information is available in the JSON crashlog, we need a way to
   tell LLDB about it. With the new symbol file format, we can easily
   synthesize a symbol file for each of those modules and load them to
   symbolicate those frames.

Here's an example of the file format:

 {
     "triple": "arm64-apple-macosx13.0.0",
     "uuid": "36D0CCE7-8ED2-3CA3-96B0-48C1764DA908",
     "symbols": [
         {
             "name": "main",
             "type": "code",
             "size": 32,
             "address": 4294983568
         },
         {
             "name": "foo",
             "type": "code",
             "size": 8,
             "address": 4294983560
         }
     ]
 }

Differential revision: https://reviews.llvm.org/D145180

(cherry picked from commit cf3524a)

Author: JDevlieghere

lldb/include/lldb/Symbol/Symbol.h

@@ -11,12 +11,23 @@
 
 #include "lldb/Core/AddressRange.h"
 #include "lldb/Core/Mangled.h"
+#include "lldb/Core/Section.h"
 #include "lldb/Symbol/SymbolContextScope.h"
 #include "lldb/Utility/UserID.h"
 #include "lldb/lldb-private.h"
+#include "llvm/Support/JSON.h"
 
 namespace lldb_private {
 
+struct JSONSymbol {
+  std::optional<uint64_t> address;
+  std::optional<uint64_t> value;
+  std::optional<uint64_t> size;
+  std::optional<uint64_t> id;
+  std::optional<lldb::SymbolType> type;
+  std::string name;
+};
+
 class Symbol : public SymbolContextScope {
 public:
   // ObjectFile readers can classify their symbol table entries and searches
@@ -39,6 +50,9 @@ class Symbol : public SymbolContextScope {
 
   const Symbol &operator=(const Symbol &rhs);
 
+  static llvm::Expected<Symbol> FromJSON(const JSONSymbol &symbol,
+                                         SectionList *section_list);
+
   void Clear();
 
   bool Compare(ConstString name, lldb::SymbolType type) const;
@@ -187,7 +201,7 @@ class Symbol : public SymbolContextScope {
 
   bool IsWeak() const { return m_is_weak; }
 
-  void SetIsWeak (bool b) { m_is_weak = b; }
+  void SetIsWeak(bool b) { m_is_weak = b; }
 
   bool GetByteSizeIsValid() const { return m_size_is_valid; }
 
@@ -332,4 +346,16 @@ class Symbol : public SymbolContextScope {
 
 } // namespace lldb_private
 
+namespace llvm {
+namespace json {
+
+bool fromJSON(const llvm::json::Value &value, lldb_private::JSONSymbol &symbol,
+              llvm::json::Path path);
+
+bool fromJSON(const llvm::json::Value &value, lldb::SymbolType &type,
+              llvm::json::Path path);
+
+} // namespace json
+} // namespace llvm
+
 #endif // LLDB_SYMBOL_SYMBOL_H

lldb/source/Plugins/ObjectFile/CMakeLists.txt

@@ -1,6 +1,7 @@
 add_subdirectory(Breakpad)
 add_subdirectory(ELF)
 add_subdirectory(JIT)
+add_subdirectory(JSON)
 add_subdirectory(Mach-O)
 add_subdirectory(Minidump)
 add_subdirectory(PDB)

lldb/source/Plugins/ObjectFile/JSON/CMakeLists.txt

@@ -0,0 +1,12 @@
+add_lldb_library(lldbPluginObjectFileJSON PLUGIN
+  ObjectFileJSON.cpp
+
+  LINK_LIBS
+    lldbCore
+    lldbHost
+    lldbSymbol
+    lldbUtility
+  LINK_COMPONENTS
+    Support
+    TargetParser
+  )

lldb/source/Plugins/ObjectFile/JSON/ObjectFileJSON.cpp

@@ -0,0 +1,176 @@
+//===-- ObjectFileJSON.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Plugins/ObjectFile/JSON/ObjectFileJSON.h"
+#include "lldb/Core/Module.h"
+#include "lldb/Core/ModuleSpec.h"
+#include "lldb/Core/PluginManager.h"
+#include "lldb/Core/Section.h"
+#include "lldb/Symbol/Symbol.h"
+#include "lldb/Utility/LLDBLog.h"
+#include "lldb/Utility/Log.h"
+#include "llvm/ADT/DenseSet.h"
+#include <optional>
+
+using namespace llvm;
+using namespace lldb;
+using namespace lldb_private;
+
+LLDB_PLUGIN_DEFINE(ObjectFileJSON)
+
+char ObjectFileJSON::ID;
+
+void ObjectFileJSON::Initialize() {
+  PluginManager::RegisterPlugin(GetPluginNameStatic(),
+                                GetPluginDescriptionStatic(), CreateInstance,
+                                CreateMemoryInstance, GetModuleSpecifications);
+}
+
+void ObjectFileJSON::Terminate() {
+  PluginManager::UnregisterPlugin(CreateInstance);
+}
+
+ObjectFile *
+ObjectFileJSON::CreateInstance(const ModuleSP &module_sp, DataBufferSP data_sp,
+                               offset_t data_offset, const FileSpec *file,
+                               offset_t file_offset, offset_t length) {
+  if (!data_sp) {
+    data_sp = MapFileData(*file, length, file_offset);
+    if (!data_sp)
+      return nullptr;
+    data_offset = 0;
+  }
+
+  if (!MagicBytesMatch(data_sp, 0, data_sp->GetByteSize()))
+    return nullptr;
+
+  if (data_sp->GetByteSize() < length) {
+    data_sp = MapFileData(*file, length, file_offset);
+    if (!data_sp)
+      return nullptr;
+    data_offset = 0;
+  }
+
+  auto text =
+      llvm::StringRef(reinterpret_cast<const char *>(data_sp->GetBytes()));
+
+  Expected<json::Value> json = json::parse(text);
+  if (!json) {
+    llvm::consumeError(json.takeError());
+    return nullptr;
+  }
+
+  json::Path::Root root;
+  Header header;
+  if (!fromJSON(*json, header, root))
+    return nullptr;
+
+  ArchSpec arch(header.triple);
+  UUID uuid;
+  uuid.SetFromStringRef(header.uuid);
+
+  Body body;
+  fromJSON(*json, body, root);
+
+  return new ObjectFileJSON(module_sp, data_sp, data_offset, file, file_offset,
+                            length, std::move(arch), std::move(uuid),
+                            std::move(body.symbols));
+}
+
+ObjectFile *ObjectFileJSON::CreateMemoryInstance(const ModuleSP &module_sp,
+                                                 WritableDataBufferSP data_sp,
+                                                 const ProcessSP &process_sp,
+                                                 addr_t header_addr) {
+  return nullptr;
+}
+
+size_t ObjectFileJSON::GetModuleSpecifications(
+    const FileSpec &file, DataBufferSP &data_sp, offset_t data_offset,
+    offset_t file_offset, offset_t length, ModuleSpecList &specs) {
+
+  if (!MagicBytesMatch(data_sp, data_offset, data_sp->GetByteSize()))
+    return 0;
+
+  auto text =
+      llvm::StringRef(reinterpret_cast<const char *>(data_sp->GetBytes()));
+
+  Expected<json::Value> json = json::parse(text);
+  if (!json) {
+    llvm::consumeError(json.takeError());
+    return 0;
+  }
+
+  json::Path::Root root;
+  Header header;
+  if (!fromJSON(*json, header, root))
+    return 0;
+
+  ArchSpec arch(header.triple);
+  UUID uuid;
+  uuid.SetFromStringRef(header.uuid);
+
+  ModuleSpec spec(file, std::move(arch));
+  spec.GetUUID() = std::move(uuid);
+  specs.Append(spec);
+  return 1;
+}
+
+ObjectFileJSON::ObjectFileJSON(const ModuleSP &module_sp, DataBufferSP &data_sp,
+                               offset_t data_offset, const FileSpec *file,
+                               offset_t offset, offset_t length, ArchSpec arch,
+                               UUID uuid, std::vector<JSONSymbol> symbols)
+    : ObjectFile(module_sp, file, offset, length, data_sp, data_offset),
+      m_arch(std::move(arch)), m_uuid(std::move(uuid)),
+      m_symbols(std::move(symbols)) {}
+
+bool ObjectFileJSON::ParseHeader() {
+  // We already parsed the header during initialization.
+  return true;
+}
+
+void ObjectFileJSON::ParseSymtab(Symtab &symtab) {
+  Log *log = GetLog(LLDBLog::Symbols);
+  SectionList *section_list = GetModule()->GetSectionList();
+  for (JSONSymbol json_symbol : m_symbols) {
+    llvm::Expected<Symbol> symbol = Symbol::FromJSON(json_symbol, section_list);
+    if (!symbol) {
+      LLDB_LOG_ERROR(log, symbol.takeError(), "invalid symbol");
+      continue;
+    }
+    symtab.AddSymbol(*symbol);
+  }
+  symtab.Finalize();
+}
+
+void ObjectFileJSON::CreateSections(SectionList &unified_section_list) {}
+
+bool ObjectFileJSON::MagicBytesMatch(DataBufferSP data_sp,
+                                     lldb::addr_t data_offset,
+                                     lldb::addr_t data_length) {
+  DataExtractor data;
+  data.SetData(data_sp, data_offset, data_length);
+  lldb::offset_t offset = 0;
+  uint32_t magic = data.GetU8(&offset);
+  return magic == '{';
+}
+
+namespace lldb_private {
+
+bool fromJSON(const json::Value &value, ObjectFileJSON::Header &header,
+              json::Path path) {
+  json::ObjectMapper o(value, path);
+  return o && o.map("triple", header.triple) && o.map("uuid", header.uuid);
+}
+
+bool fromJSON(const json::Value &value, ObjectFileJSON::Body &body,
+              json::Path path) {
+  json::ObjectMapper o(value, path);
+  return o && o.map("symbols", body.symbols);
+}
+
+} // namespace lldb_private

lldb/source/Plugins/ObjectFile/JSON/ObjectFileJSON.h

@@ -0,0 +1,119 @@
+//===-- ObjectFileJSON.h -------------------------------------- -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_SOURCE_PLUGINS_OBJECTFILE_JSON_OBJECTFILEJSON_H
+#define LLDB_SOURCE_PLUGINS_OBJECTFILE_JSON_OBJECTFILEJSON_H
+
+#include "lldb/Symbol/ObjectFile.h"
+#include "lldb/Utility/ArchSpec.h"
+#include "llvm/Support/JSON.h"
+
+namespace lldb_private {
+
+class ObjectFileJSON : public ObjectFile {
+public:
+  static void Initialize();
+  static void Terminate();
+
+  static llvm::StringRef GetPluginNameStatic() { return "JSON"; }
+
+  static const char *GetPluginDescriptionStatic() {
+    return "JSON object file reader.";
+  }
+
+  static ObjectFile *
+  CreateInstance(const lldb::ModuleSP &module_sp, lldb::DataBufferSP data_sp,
+                 lldb::offset_t data_offset, const FileSpec *file,
+                 lldb::offset_t file_offset, lldb::offset_t length);
+
+  static ObjectFile *CreateMemoryInstance(const lldb::ModuleSP &module_sp,
+                                          lldb::WritableDataBufferSP data_sp,
+                                          const lldb::ProcessSP &process_sp,
+                                          lldb::addr_t header_addr);
+
+  static size_t GetModuleSpecifications(const FileSpec &file,
+                                        lldb::DataBufferSP &data_sp,
+                                        lldb::offset_t data_offset,
+                                        lldb::offset_t file_offset,
+                                        lldb::offset_t length,
+                                        ModuleSpecList &specs);
+
+  llvm::StringRef GetPluginName() override { return GetPluginNameStatic(); }
+
+  // LLVM RTTI support
+  static char ID;
+  bool isA(const void *ClassID) const override {
+    return ClassID == &ID || ObjectFile::isA(ClassID);
+  }
+  static bool classof(const ObjectFile *obj) { return obj->isA(&ID); }
+
+  bool ParseHeader() override;
+
+  lldb::ByteOrder GetByteOrder() const override {
+    return m_arch.GetByteOrder();
+  }
+
+  bool IsExecutable() const override { return false; }
+
+  uint32_t GetAddressByteSize() const override {
+    return m_arch.GetAddressByteSize();
+  }
+
+  AddressClass GetAddressClass(lldb::addr_t file_addr) override {
+    return AddressClass::eInvalid;
+  }
+
+  void ParseSymtab(lldb_private::Symtab &symtab) override;
+
+  bool IsStripped() override { return false; }
+
+  void CreateSections(SectionList &unified_section_list) override;
+
+  void Dump(Stream *s) override {}
+
+  ArchSpec GetArchitecture() override { return m_arch; }
+
+  UUID GetUUID() override { return m_uuid; }
+
+  uint32_t GetDependentModules(FileSpecList &files) override { return 0; }
+
+  Type CalculateType() override { return eTypeDebugInfo; }
+
+  Strata CalculateStrata() override { return eStrataUser; }
+
+  static bool MagicBytesMatch(lldb::DataBufferSP data_sp, lldb::addr_t offset,
+                              lldb::addr_t length);
+
+  struct Header {
+    std::string triple;
+    std::string uuid;
+  };
+
+  struct Body {
+    std::vector<JSONSymbol> symbols;
+  };
+
+private:
+  ArchSpec m_arch;
+  UUID m_uuid;
+  std::vector<JSONSymbol> m_symbols;
+
+  ObjectFileJSON(const lldb::ModuleSP &module_sp, lldb::DataBufferSP &data_sp,
+                 lldb::offset_t data_offset, const FileSpec *file,
+                 lldb::offset_t offset, lldb::offset_t length, ArchSpec arch,
+                 UUID uuid, std::vector<JSONSymbol> symbols);
+};
+
+bool fromJSON(const llvm::json::Value &value, ObjectFileJSON::Header &header,
+              llvm::json::Path path);
+
+bool fromJSON(const llvm::json::Value &value, ObjectFileJSON::Body &body,
+              llvm::json::Path path);
+
+} // namespace lldb_private
+#endif // LLDB_SOURCE_PLUGINS_OBJECTFILE_JSON_OBJECTFILEJSON_H

lldb/source/Plugins/SymbolFile/CMakeLists.txt

@@ -1,5 +1,6 @@
 add_subdirectory(Breakpad)
 add_subdirectory(DWARF)
+add_subdirectory(JSON)
 add_subdirectory(NativePDB)
 add_subdirectory(PDB)
 add_subdirectory(Symtab)

lldb/source/Plugins/SymbolFile/JSON/CMakeLists.txt

@@ -0,0 +1,7 @@
+add_lldb_library(lldbPluginSymbolFileJSON PLUGIN
+  SymbolFileJSON.cpp
+
+  LINK_LIBS
+    lldbCore
+    lldbSymbol
+  )