Skip to content

[lldb] Add count for number of DWO files loaded in statistics #144424

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jun 23, 2025
Merged

[lldb] Add count for number of DWO files loaded in statistics #144424

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
57483ee
Add counter for number of DWO files loaded in statistics
qxy11 Jun 16, 2025
3292b89
Use GetSeparateDebugInfo and add totalLoadedDwoFileCount
qxy11 Jun 17, 2025
a90294d
Put file counting logic in UpdateDwoFileCounts and add test_debug_nam…
qxy11 Jun 17, 2025
9b96648
Add dwo_file_count and loaded_dwo_file_count to ModuleStats
qxy11 Jun 18, 2025
d3e89b7
Add a separate GetDwoFileCounts instead of using GetSeparateDebugInfo
qxy11 Jun 18, 2025
ef1c131
Fix a typo in the comments
qxy11 Jun 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion lldb/include/lldb/Symbol/SymbolFile.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -467,8 +467,12 @@ class SymbolFile : public PluginInterface {
/// If true, then only return separate debug info files that encountered
/// errors during loading. If false, then return all expected separate
/// debug info files, regardless of whether they were successfully loaded.
/// \param load_all_debug_info
/// If true, force loading any symbol files if they are not yet loaded and
/// add its info to the debug info files. Default to true.
virtual bool GetSeparateDebugInfo(StructuredData::Dictionary &d,
bool errors_only) {
bool errors_only,
bool load_all_debug_info = true) {
return false;
};

Expand Down
7 changes: 4 additions & 3 deletions lldb/include/lldb/Symbol/SymbolFileOnDemand.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,9 +223,10 @@ class SymbolFileOnDemand : public lldb_private::SymbolFile {
return m_sym_file_impl->SetDebugInfoHadFrameVariableErrors();
}

bool GetSeparateDebugInfo(StructuredData::Dictionary &d,
bool errors_only) override {
return m_sym_file_impl->GetSeparateDebugInfo(d, errors_only);
bool GetSeparateDebugInfo(StructuredData::Dictionary &d, bool errors_only,
bool load_all_debug_info = true) override {
return m_sym_file_impl->GetSeparateDebugInfo(d, errors_only,
load_all_debug_info);
}

lldb::TypeSP MakeType(lldb::user_id_t uid, ConstString name,
Expand Down
2 changes: 2 additions & 0 deletions lldb/include/lldb/Target/Statistics.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,8 @@ struct ModuleStats {
bool symtab_stripped = false;
bool debug_info_had_variable_errors = false;
bool debug_info_had_incomplete_types = false;
uint32_t dwo_file_count = 0;
uint32_t loaded_dwo_file_count = 0;
};

struct ConstStringStats {
Expand Down
26 changes: 19 additions & 7 deletions lldb/packages/Python/lldbsuite/test/builders/builder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,13 +247,25 @@ def getLLDBObjRoot(self):
def _getDebugInfoArgs(self, debug_info):
if debug_info is None:
return []
if debug_info == "dwarf":
return ["MAKE_DSYM=NO"]
if debug_info == "dwo":
return ["MAKE_DSYM=NO", "MAKE_DWO=YES"]
if debug_info == "gmodules":
return ["MAKE_DSYM=NO", "MAKE_GMODULES=YES"]
return None

debug_options = debug_info if isinstance(debug_info, list) else [debug_info]
option_flags = {
"dwarf": {"MAKE_DSYM": "NO"},
"dwo": {"MAKE_DSYM": "NO", "MAKE_DWO": "YES"},
"gmodules": {"MAKE_DSYM": "NO", "MAKE_GMODULES": "YES"},
"debug_names": {"MAKE_DEBUG_NAMES": "YES"},
"dwp": {"MAKE_DSYM": "NO", "MAKE_DWP": "YES"},
}

# Collect all flags, with later options overriding earlier ones
flags = {}

for option in debug_options:
if not option or option not in option_flags:
return None # Invalid options
flags.update(option_flags[option])

return [f"{key}={value}" for key, value in flags.items()]

def getBuildCommand(
self,
Expand Down
4 changes: 4 additions & 0 deletions lldb/packages/Python/lldbsuite/test/make/Makefile.rules
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,10 @@ ifeq "$(MAKE_DWO)" "YES"
CFLAGS += -gsplit-dwarf
endif

ifeq "$(MAKE_DEBUG_NAMES)" "YES"
CFLAGS += -gpubnames
endif

ifeq "$(USE_PRIVATE_MODULE_CACHE)" "YES"
THE_CLANG_MODULE_CACHE_DIR := $(BUILDDIR)/private-module-cache
else
Expand Down
5 changes: 3 additions & 2 deletions lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4139,7 +4139,8 @@ void SymbolFileDWARF::DumpClangAST(Stream &s, llvm::StringRef filter) {
}

bool SymbolFileDWARF::GetSeparateDebugInfo(StructuredData::Dictionary &d,
bool errors_only) {
bool errors_only,
bool load_all_debug_info) {
StructuredData::Array separate_debug_info_files;
DWARFDebugInfo &info = DebugInfo();
const size_t num_cus = info.GetNumUnits();
Expand Down Expand Up @@ -4182,7 +4183,7 @@ bool SymbolFileDWARF::GetSeparateDebugInfo(StructuredData::Dictionary &d,

// If we have a DWO symbol file, that means we were able to successfully
// load it.
SymbolFile *dwo_symfile = dwarf_cu->GetDwoSymbolFile();
SymbolFile *dwo_symfile = dwarf_cu->GetDwoSymbolFile(load_all_debug_info);
if (dwo_symfile) {
dwo_data->AddStringItem(
"resolved_dwo_path",
Expand Down
4 changes: 2 additions & 2 deletions lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,8 +279,8 @@ class SymbolFileDWARF : public SymbolFileCommon {
void DumpClangAST(Stream &s, llvm::StringRef filter) override;

/// List separate dwo files.
bool GetSeparateDebugInfo(StructuredData::Dictionary &d,
bool errors_only) override;
bool GetSeparateDebugInfo(StructuredData::Dictionary &d, bool errors_only,
bool load_all_debug_info = true) override;

DWARFContext &GetDWARFContext() { return m_context; }

Expand Down
3 changes: 2 additions & 1 deletion lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1278,7 +1278,8 @@ void SymbolFileDWARFDebugMap::DumpClangAST(Stream &s, llvm::StringRef filter) {
}

bool SymbolFileDWARFDebugMap::GetSeparateDebugInfo(
lldb_private::StructuredData::Dictionary &d, bool errors_only) {
lldb_private::StructuredData::Dictionary &d, bool errors_only,
bool load_all_debug_info) {
StructuredData::Array separate_debug_info_files;
const uint32_t cu_count = GetNumCompileUnits();
for (uint32_t cu_idx = 0; cu_idx < cu_count; ++cu_idx) {
Expand Down
4 changes: 2 additions & 2 deletions lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,8 +132,8 @@ class SymbolFileDWARFDebugMap : public SymbolFileCommon {
void DumpClangAST(Stream &s, llvm::StringRef filter) override;

/// List separate oso files.
bool GetSeparateDebugInfo(StructuredData::Dictionary &d,
bool errors_only) override;
bool GetSeparateDebugInfo(StructuredData::Dictionary &d, bool errors_only,
bool load_all_debug_info = true) override;

// PluginInterface protocol
llvm::StringRef GetPluginName() override { return GetPluginNameStatic(); }
Expand Down
43 changes: 42 additions & 1 deletion lldb/source/Target/Statistics.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,37 @@ static void EmplaceSafeString(llvm::json::Object &obj, llvm::StringRef key,
obj.try_emplace(key, llvm::json::fixUTF8(str));
}

static void UpdateDwoFileCounts(SymbolFile *sym_file,
uint32_t &total_dwo_file_count,
uint32_t &total_loaded_dwo_file_count) {
// Count DWO files from this symbol file using GetSeparateDebugInfo
// For DWP files, this increments counts for both total and successfully
// loaded DWO CUs. For non split-dwarf files, these counts should not change
StructuredData::Dictionary separate_debug_info;
if (sym_file->GetSeparateDebugInfo(separate_debug_info,
Copy link
Contributor

@jeffreytan81 jeffreytan81 Jun 18, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would not recommend calling GetSeparateDebugInfo API for this task. GetSeparateDebugInfo API constructs StructuredData::Dictionary JSON style objects internally which is not cheap (I have seen profile traces that indicate constructing StructuredData::Dictionary as hot paths). This can get worse considering we are using statistics dump in lldb session logging by default. (means we are paying the cost every time).

If we want to reuse it, I would suggest refactoring and reuse the necessary underlying implementation without extra expensive burdens.

Copy link
Contributor Author

@qxy11 qxy11 Jun 18, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Makes sense, I'll make a separate API for getting the counts here without constructing StructuredData::Dictionary.

/*errors_only=*/false,
/*load_all_debug_info=*/false)) {
llvm::StringRef type;
if (separate_debug_info.GetValueForKeyAsString("type", type) &&
type == "dwo") {
StructuredData::Array *files;
if (separate_debug_info.GetValueForKeyAsArray("separate-debug-info-files",
files)) {
files->ForEach([&](StructuredData::Object *obj) {
if (auto dict = obj->GetAsDictionary()) {
total_dwo_file_count++;

bool loaded = false;
if (dict->GetValueForKeyAsBoolean("loaded", loaded) && loaded)
Copy link
Contributor

@jeffreytan81 jeffreytan81 Jun 18, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am not sure this is correct for dwp. In dwp scenario, there is one dwp file for the entire module (instead of one dwo for each compile unit), so only one SymbolFileDWARFDwo will be created. Then, if you check the code here https://github.com/llvm/llvm-project/blob/main/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp#L4185-L4194, it will mark that dwo as "loaded if SymbolFileDWARFDwo exists, which will happen for all dwo files during dwp scenarios which can be wrong -- if you have .debug_names, lldb won't need to parse all debug info during startup but parse/load lazily so only partial of dwp are parsed while the current implementation reports all parsed/loaded. This is probably a bug in original SymbolFileDWARF::GetSeparateDebugInfo implementation though, cc @zhyty

Copy link
Contributor Author

@qxy11 qxy11 Jun 18, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should work for DWP and correctly report the # of parsed CUs (see the unit test added test_dwp_dwo_file_count here).

My understanding is that each CU has its own m_dwo shared_ptr object rather than there being a global DWP reference that's shared. It looks like the DWP file first gets loaded here, the specific DWO unit gets indexed here, and then only after the specific DWO data for the CU is parsed does m_dwo get set. So in the code for GetSeparateDebugInfo it should get the correct per-unit counts.

total_loaded_dwo_file_count++;
}
return true;
});
}
}
}
}

json::Value StatsSuccessFail::ToJSON() const {
return json::Object{{"successes", successes}, {"failures", failures}};
}
Expand Down Expand Up @@ -73,6 +104,8 @@ json::Value ModuleStats::ToJSON() const {
debug_info_had_incomplete_types);
module.try_emplace("symbolTableStripped", symtab_stripped);
module.try_emplace("symbolTableSymbolCount", symtab_symbol_count);
module.try_emplace("dwoFileCount", dwo_file_count);
module.try_emplace("loadedDwoFileCount", loaded_dwo_file_count);

if (!symbol_locator_time.map.empty()) {
json::Object obj;
Expand All @@ -86,7 +119,7 @@ json::Value ModuleStats::ToJSON() const {

if (!symfile_modules.empty()) {
json::Array symfile_ids;
for (const auto symfile_id: symfile_modules)
for (const auto symfile_id : symfile_modules)
symfile_ids.emplace_back(symfile_id);
module.try_emplace("symbolFileModuleIdentifiers", std::move(symfile_ids));
}
Expand Down Expand Up @@ -322,6 +355,8 @@ llvm::json::Value DebuggerStats::ReportStatistics(
uint32_t num_modules_with_incomplete_types = 0;
uint32_t num_stripped_modules = 0;
uint32_t symtab_symbol_count = 0;
uint32_t total_loaded_dwo_file_count = 0;
uint32_t total_dwo_file_count = 0;
for (size_t image_idx = 0; image_idx < num_modules; ++image_idx) {
Module *module = target != nullptr
? target->GetImages().GetModuleAtIndex(image_idx).get()
Expand Down Expand Up @@ -353,6 +388,10 @@ llvm::json::Value DebuggerStats::ReportStatistics(
for (const auto &symbol_module : symbol_modules.Modules())
module_stat.symfile_modules.push_back((intptr_t)symbol_module.get());
}
UpdateDwoFileCounts(sym_file, module_stat.dwo_file_count,
module_stat.loaded_dwo_file_count);
total_dwo_file_count += module_stat.dwo_file_count;
total_loaded_dwo_file_count += module_stat.loaded_dwo_file_count;
module_stat.debug_info_index_loaded_from_cache =
sym_file->GetDebugInfoIndexWasLoadedFromCache();
if (module_stat.debug_info_index_loaded_from_cache)
Expand Down Expand Up @@ -427,6 +466,8 @@ llvm::json::Value DebuggerStats::ReportStatistics(
{"totalDebugInfoEnabled", num_debug_info_enabled_modules},
{"totalSymbolTableStripped", num_stripped_modules},
{"totalSymbolTableSymbolCount", symtab_symbol_count},
{"totalLoadedDwoFileCount", total_loaded_dwo_file_count},
{"totalDwoFileCount", total_dwo_file_count},
};

if (include_targets) {
Expand Down
Loading