Report only loaded debug info in statistics dump (llvm#81706)

kusmour · web-flow · commit 21ddd7ff2b16 · 2024-02-17T14:38:18.000-05:00
Currently running `statistics dump` will trigger lldb to load debug info
that's not yet loaded (eg. dwo files). Resulted in a delay in the
command return, which, can be interrupting.

This patch also added a new option `--load-all-debug-info` asking
statistics to dump all possible debug info, which will force loading all
debug info available if not yet loaded.
diff --git a/lldb/bindings/interface/SBStatisticsOptionsDocstrings.i b/lldb/bindings/interface/SBStatisticsOptionsDocstrings.i
@@ -6,3 +6,9 @@
 ) lldb::SBStatisticsOptions::SetSummaryOnly;
 %feature("docstring", "Gets whether the statistics only dump a summary."
 ) lldb::SBStatisticsOptions::GetSummaryOnly;
+%feature("docstring", "
+    Sets whether the statistics will force loading all possible debug info."
+) lldb::SBStatisticsOptions::SetReportAllAvailableDebugInfo;
+%feature("docstring", "
+    Gets whether the statistics will force loading all possible debug info."
+) lldb::SBStatisticsOptions::GetReportAllAvailableDebugInfo;
diff --git a/lldb/include/lldb/API/SBStatisticsOptions.h b/lldb/include/lldb/API/SBStatisticsOptions.h
@@ -25,6 +25,14 @@ class LLDB_API SBStatisticsOptions {
   void SetSummaryOnly(bool b);
   bool GetSummaryOnly();
 
+  /// If set to true, the debugger will load all debug info that is available
+  /// and report statistics on the total amount. If this is set to false, then
+  /// only report statistics on the currently loaded debug information.
+  /// This can avoid loading debug info from separate files just so it can
+  /// report the total size which can slow down statistics reporting.
+  void SetReportAllAvailableDebugInfo(bool b);
+  bool GetReportAllAvailableDebugInfo();
+
 protected:
   friend class SBTarget;
   const lldb_private::StatisticsOptions &ref() const;
diff --git a/lldb/include/lldb/Symbol/SymbolFile.h b/lldb/include/lldb/Symbol/SymbolFile.h
@@ -381,7 +381,8 @@ class SymbolFile : public PluginInterface {
 
   /// Metrics gathering functions
 
-  /// Return the size in bytes of all debug information in the symbol file.
+  /// Return the size in bytes of all loaded debug information or total possible
+  /// debug info in the symbol file.
   ///
   /// If the debug information is contained in sections of an ObjectFile, then
   /// this call should add the size of all sections that contain debug
@@ -391,7 +392,14 @@ class SymbolFile : public PluginInterface {
   /// entire file should be returned. The default implementation of this
   /// function will iterate over all sections in a module and add up their
   /// debug info only section byte sizes.
-  virtual uint64_t GetDebugInfoSize() = 0;
+  ///
+  /// \param load_all_debug_info
+  ///   If true, force loading any symbol files if they are not yet loaded and
+  ///   add to the total size. Default to false.
+  ///
+  /// \returns
+  ///   Total currently loaded debug info size in bytes
+  virtual uint64_t GetDebugInfoSize(bool load_all_debug_info = false) = 0;
 
   /// Return the time taken to parse the debug information.
   ///
@@ -534,7 +542,7 @@ class SymbolFileCommon : public SymbolFile {
 
   void Dump(Stream &s) override;
 
-  uint64_t GetDebugInfoSize() override;
+  uint64_t GetDebugInfoSize(bool load_all_debug_info = false) override;
 
   bool GetDebugInfoIndexWasLoadedFromCache() const override {
     return m_index_was_loaded_from_cache;
diff --git a/lldb/include/lldb/Symbol/SymbolFileOnDemand.h b/lldb/include/lldb/Symbol/SymbolFileOnDemand.h
@@ -178,7 +178,7 @@ class SymbolFileOnDemand : public lldb_private::SymbolFile {
 
   void PreloadSymbols() override;
 
-  uint64_t GetDebugInfoSize() override;
+  uint64_t GetDebugInfoSize(bool load_all_debug_info = false) override;
   lldb_private::StatsDuration::Duration GetDebugInfoParseTime() override;
   lldb_private::StatsDuration::Duration GetDebugInfoIndexTime() override;
 
diff --git a/lldb/include/lldb/Target/Statistics.h b/lldb/include/lldb/Target/Statistics.h
@@ -132,6 +132,7 @@ struct ConstStringStats {
 
 struct StatisticsOptions {
   bool summary_only = false;
+  bool load_all_debug_info = false;
 };
 
 /// A class that represents statistics for a since lldb_private::Target.
diff --git a/lldb/source/API/SBStatisticsOptions.cpp b/lldb/source/API/SBStatisticsOptions.cpp
@@ -44,6 +44,14 @@ void SBStatisticsOptions::SetSummaryOnly(bool b) {
 
 bool SBStatisticsOptions::GetSummaryOnly() { return m_opaque_up->summary_only; }
 
+void SBStatisticsOptions::SetReportAllAvailableDebugInfo(bool b) {
+  m_opaque_up->load_all_debug_info = b;
+}
+
+bool SBStatisticsOptions::GetReportAllAvailableDebugInfo() {
+  return m_opaque_up->load_all_debug_info;
+}
+
 const lldb_private::StatisticsOptions &SBStatisticsOptions::ref() const {
   return *m_opaque_up;
 }
diff --git a/lldb/source/Commands/CommandObjectStats.cpp b/lldb/source/Commands/CommandObjectStats.cpp
@@ -78,6 +78,9 @@ class CommandObjectStatsDump : public CommandObjectParsed {
       case 's':
         m_stats_options.summary_only = true;
         break;
+      case 'f':
+        m_stats_options.load_all_debug_info = true;
+        break;
       default:
         llvm_unreachable("Unimplemented option");
       }
diff --git a/lldb/source/Commands/Options.td b/lldb/source/Commands/Options.td
@@ -1419,6 +1419,10 @@ let Command = "statistics dump" in {
   def statistics_dump_all: Option<"all-targets", "a">, Group<1>,
     Desc<"Include statistics for all targets.">;
   def statistics_dump_summary: Option<"summary", "s">, Group<1>,
-    Desc<"Dump only high-level summary statistics."
+    Desc<"Dump only high-level summary statistics. "
          "Exclude targets, modules, breakpoints etc... details.">;
+  def statistics_dump_force: Option<"load-all-debug-info", "f">, Group<1>,
+    Desc<"Dump the total possible debug info statistics. "
+    "Force loading all the debug information if not yet loaded, and collect "
+    "statistics with those.">;
 }
diff --git a/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.cpp b/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.cpp
@@ -918,7 +918,7 @@ void SymbolFileBreakpad::ParseUnwindData() {
   m_unwind_data->win.Sort();
 }
 
-uint64_t SymbolFileBreakpad::GetDebugInfoSize() {
+uint64_t SymbolFileBreakpad::GetDebugInfoSize(bool load_all_debug_info) {
   // Breakpad files are all debug info.
   return m_objfile_sp->GetByteSize();
 }
diff --git a/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.h b/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.h
@@ -141,7 +141,7 @@ class SymbolFileBreakpad : public SymbolFileCommon {
 
   llvm::StringRef GetPluginName() override { return GetPluginNameStatic(); }
 
-  uint64_t GetDebugInfoSize() override;
+  uint64_t GetDebugInfoSize(bool load_all_debug_info = false) override;
 
 private:
   // A class representing a position in the breakpad file. Useful for
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.cpp
@@ -896,8 +896,9 @@ void DWARFUnit::ComputeAbsolutePath() {
     m_file_spec->MakeAbsolute(GetCompilationDirectory());
 }
 
-SymbolFileDWARFDwo *DWARFUnit::GetDwoSymbolFile() {
-  ExtractUnitDIEIfNeeded();
+SymbolFileDWARFDwo *DWARFUnit::GetDwoSymbolFile(bool load_all_debug_info) {
+  if (load_all_debug_info)
+    ExtractUnitDIEIfNeeded();
   if (m_dwo)
     return &llvm::cast<SymbolFileDWARFDwo>(m_dwo->GetSymbolFileDWARF());
   return nullptr;
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFUnit.h
@@ -241,7 +241,7 @@ class DWARFUnit : public UserID {
   FileSpec GetFile(size_t file_idx);
   FileSpec::Style GetPathStyle();
 
-  SymbolFileDWARFDwo *GetDwoSymbolFile();
+  SymbolFileDWARFDwo *GetDwoSymbolFile(bool load_all_debug_info = false);
 
   die_iterator_range dies() {
     ExtractDIEsIfNeeded();
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
@@ -2672,7 +2672,7 @@ static bool UpdateCompilerContextForSimpleTemplateNames(TypeQuery &match) {
   return any_context_updated;
 }
 
-uint64_t SymbolFileDWARF::GetDebugInfoSize() {
+uint64_t SymbolFileDWARF::GetDebugInfoSize(bool load_all_debug_info) {
   DWARFDebugInfo &info = DebugInfo();
   uint32_t num_comp_units = info.GetNumUnits();
 
@@ -2687,7 +2687,7 @@ uint64_t SymbolFileDWARF::GetDebugInfoSize() {
     if (cu == nullptr)
       continue;
 
-    SymbolFileDWARFDwo *dwo = cu->GetDwoSymbolFile();
+    SymbolFileDWARFDwo *dwo = cu->GetDwoSymbolFile(load_all_debug_info);
     if (dwo)
       debug_info_size += dwo->GetDebugInfoSize();
   }
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
@@ -186,7 +186,7 @@ class SymbolFileDWARF : public SymbolFileCommon {
   GetMangledNamesForFunction(const std::string &scope_qualified_name,
                              std::vector<ConstString> &mangled_names) override;
 
-  uint64_t GetDebugInfoSize() override;
+  uint64_t GetDebugInfoSize(bool load_all_debug_info = false) override;
 
   void FindTypes(const lldb_private::TypeQuery &match,
                  lldb_private::TypeResults &results) override;
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.cpp
@@ -85,7 +85,7 @@ lldb::offset_t SymbolFileDWARFDwo::GetVendorDWARFOpcodeSize(
   return GetBaseSymbolFile().GetVendorDWARFOpcodeSize(data, data_offset, op);
 }
 
-uint64_t SymbolFileDWARFDwo::GetDebugInfoSize() {
+uint64_t SymbolFileDWARFDwo::GetDebugInfoSize(bool load_all_debug_info) {
   // Directly get debug info from current dwo object file's section list
   // instead of asking SymbolFileCommon::GetDebugInfo() which parses from
   // owning module which is wrong.
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.h b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDwo.h
@@ -47,7 +47,7 @@ class SymbolFileDWARFDwo : public SymbolFileDWARF {
                                           const lldb::offset_t data_offset,
                                           const uint8_t op) const override;
 
-  uint64_t GetDebugInfoSize() override;
+  uint64_t GetDebugInfoSize(bool load_all_debug_info = false) override;
 
   bool ParseVendorDWARFOpcode(uint8_t op, const DataExtractor &opcodes,
                               lldb::offset_t &offset,
diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp
@@ -2156,7 +2156,7 @@ SymbolFileNativePDB::GetTypeSystemForLanguage(lldb::LanguageType language) {
   return type_system_or_err;
 }
 
-uint64_t SymbolFileNativePDB::GetDebugInfoSize() {
+uint64_t SymbolFileNativePDB::GetDebugInfoSize(bool load_all_debug_info) {
   // PDB files are a separate file that contains all debug info.
   return m_index->pdb().getFileSize();
 }
diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.h b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.h
@@ -77,7 +77,7 @@ class SymbolFileNativePDB : public SymbolFileCommon {
 
   void InitializeObject() override;
 
-  uint64_t GetDebugInfoSize() override;
+  uint64_t GetDebugInfoSize(bool load_all_debug_info = false) override;
 
   // Compile Unit function calls
 
diff --git a/lldb/source/Symbol/SymbolFile.cpp b/lldb/source/Symbol/SymbolFile.cpp
@@ -227,7 +227,7 @@ SymbolFileCommon::GetTypeSystemForLanguage(lldb::LanguageType language) {
   return type_system_or_err;
 }
 
-uint64_t SymbolFileCommon::GetDebugInfoSize() {
+uint64_t SymbolFileCommon::GetDebugInfoSize(bool load_all_debug_info) {
   if (!m_objfile_sp)
     return 0;
   ModuleSP module_sp(m_objfile_sp->GetModule());
diff --git a/lldb/source/Symbol/SymbolFileOnDemand.cpp b/lldb/source/Symbol/SymbolFileOnDemand.cpp
@@ -535,11 +535,11 @@ void SymbolFileOnDemand::PreloadSymbols() {
   return m_sym_file_impl->PreloadSymbols();
 }
 
-uint64_t SymbolFileOnDemand::GetDebugInfoSize() {
+uint64_t SymbolFileOnDemand::GetDebugInfoSize(bool load_all_debug_info) {
   // Always return the real debug info size.
   LLDB_LOG(GetLog(), "[{0}] {1} is not skipped", GetSymbolFileName(),
            __FUNCTION__);
-  return m_sym_file_impl->GetDebugInfoSize();
+  return m_sym_file_impl->GetDebugInfoSize(load_all_debug_info);
 }
 
 StatsDuration::Duration SymbolFileOnDemand::GetDebugInfoParseTime() {
diff --git a/lldb/source/Target/Statistics.cpp b/lldb/source/Target/Statistics.cpp
@@ -224,6 +224,7 @@ llvm::json::Value DebuggerStats::ReportStatistics(
     const lldb_private::StatisticsOptions &options) {
 
   const bool summary_only = options.summary_only;
+  const bool load_all_debug_info = options.load_all_debug_info;
 
   json::Array json_targets;
   json::Array json_modules;
@@ -280,7 +281,8 @@ llvm::json::Value DebuggerStats::ReportStatistics(
         ++debug_index_saved;
       module_stat.debug_index_time = sym_file->GetDebugInfoIndexTime().count();
       module_stat.debug_parse_time = sym_file->GetDebugInfoParseTime().count();
-      module_stat.debug_info_size = sym_file->GetDebugInfoSize();
+      module_stat.debug_info_size =
+          sym_file->GetDebugInfoSize(load_all_debug_info);
       module_stat.symtab_stripped = module->GetObjectFile()->IsStripped();
       if (module_stat.symtab_stripped)
         ++num_stripped_modules;
diff --git a/lldb/test/API/functionalities/stats_api/TestStatisticsAPI.py b/lldb/test/API/functionalities/stats_api/TestStatisticsAPI.py
@@ -117,3 +117,38 @@ def test_command_stats_api(self):
         self.assertNotIn("bt", command_stats)
         # Verify bt's regex command is not duplicatedly captured.
         self.assertNotIn("_regexp-bt", command_stats)
+
+    def test_command_stats_force(self):
+        """
+        Test reporting all pssible debug info stats by force loading all debug
+        info. For example, dwo files
+        """
+        src_dir = self.getSourceDir()
+        dwo_yaml_path = os.path.join(src_dir, "main.dwo.yaml")
+        obj_yaml_path = os.path.join(src_dir, "main.o.yaml")
+        dwo_path = self.getBuildArtifact("main.dwo")
+        obj_path = self.getBuildArtifact("main.o")
+        self.yaml2obj(dwo_yaml_path, dwo_path)
+        self.yaml2obj(obj_yaml_path, obj_path)
+
+        # We need the current working directory to be set to the build directory
+        os.chdir(self.getBuildDir())
+        # Create a target with the object file we just created from YAML
+        target = self.dbg.CreateTarget(obj_path)
+        self.assertTrue(target, VALID_TARGET)
+
+        # Get statistics
+        stats_options = lldb.SBStatisticsOptions()
+        stats = target.GetStatistics(stats_options)
+        stream = lldb.SBStream()
+        stats.GetAsJSON(stream)
+        debug_stats = json.loads(stream.GetData())
+        self.assertEqual(debug_stats["totalDebugInfoByteSize"], 188)
+
+        # Get statistics with force loading
+        stats_options.SetReportAllAvailableDebugInfo(True)
+        stats_force = target.GetStatistics(stats_options)
+        stream_force = lldb.SBStream()
+        stats_force.GetAsJSON(stream_force)
+        debug_stats_force = json.loads(stream_force.GetData())
+        self.assertEqual(debug_stats_force["totalDebugInfoByteSize"], 435)
diff --git a/lldb/test/API/functionalities/stats_api/main.dwo.yaml b/lldb/test/API/functionalities/stats_api/main.dwo.yaml
@@ -0,0 +1,37 @@
+--- !ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_X86_64
+  SectionHeaderStringTable: .strtab
+Sections:
+  - Name:            .debug_str.dwo
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_EXCLUDE, SHF_MERGE, SHF_STRINGS ]
+    AddressAlign:    0x1
+    EntSize:         0x1
+    Content:         5F5A33666F6F7600666F6F006D61696E00696E7400636C616E672076657273696F6E2031372E302E36202843656E744F532031372E302E362D342E656C3929006D61696E2E637070006D61696E2E64776F00
+  - Name:            .debug_str_offsets.dwo
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_EXCLUDE ]
+    AddressAlign:    0x1
+    Content:         00000000080000000C00000011000000150000004000000049000000
+  - Name:            .debug_info.dwo
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_EXCLUDE ]
+    AddressAlign:    0x1
+    Content:         3500000004000000000008010421000506939F5FCB7816797B02000600000001560001010103011C0000000156020105340000000403050400
+  - Name:            .debug_abbrev.dwo
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_EXCLUDE ]
+    AddressAlign:    0x1
+    Content:         01110125823E130503823EB042823EB142070000022E0011813E120640186E823E03823E3A0B3B0B3F190000032E0011813E1206401803823E3A0B3B0B49133F19000004240003823E3E0B0B0B000000
+  - Type:            SectionHeaderTable
+    Sections:
+      - Name:            .strtab
+      - Name:            .debug_str.dwo
+      - Name:            .debug_str_offsets.dwo
+      - Name:            .debug_info.dwo
+      - Name:            .debug_abbrev.dwo
+...
diff --git a/lldb/test/API/functionalities/stats_api/main.o.yaml b/lldb/test/API/functionalities/stats_api/main.o.yaml

Original file line number	Diff line number	Diff line change
`@@ -918,7 +918,7 @@ void SymbolFileBreakpad::ParseUnwindData() {`
`918`	`918`	`m_unwind_data->win.Sort();`
`919`	`919`	`}`
`920`	`920`
`921`		`-uint64_t SymbolFileBreakpad::GetDebugInfoSize() {`
	`921`	`+uint64_t SymbolFileBreakpad::GetDebugInfoSize(bool load_all_debug_info) {`
`922`	`922`	`// Breakpad files are all debug info.`
`923`	`923`	`return m_objfile_sp->GetByteSize();`
`924`	`924`	`}`