diff --git a/lldb/include/lldb/API/SBDebugger.h b/lldb/include/lldb/API/SBDebugger.h index 84ea9c0f772e1..b12c4623de285 100644 --- a/lldb/include/lldb/API/SBDebugger.h +++ b/lldb/include/lldb/API/SBDebugger.h @@ -13,6 +13,7 @@ #include "lldb/API/SBDefines.h" #include "lldb/API/SBPlatform.h" +#include "lldb/API/SBStructuredData.h" namespace lldb_private { class CommandPluginInterfaceImplementation; @@ -245,6 +246,8 @@ class LLDB_API SBDebugger { lldb::SBTarget GetDummyTarget(); + void SendTelemetry(const lldb::SBStructuredData &entry); + // Return true if target is deleted from the target list of the debugger. bool DeleteTarget(lldb::SBTarget &target); diff --git a/lldb/include/lldb/Core/Debugger.h b/lldb/include/lldb/Core/Debugger.h index a72c2596cc2c5..82f0806e96e3b 100644 --- a/lldb/include/lldb/Core/Debugger.h +++ b/lldb/include/lldb/Core/Debugger.h @@ -19,6 +19,8 @@ #include "lldb/Core/FormatEntity.h" #include "lldb/Core/IOHandler.h" #include "lldb/Core/SourceManager.h" +#include "lldb/Core/StructuredDataImpl.h" +#include "lldb/Core/Telemetry.h" #include "lldb/Core/UserSettingsController.h" #include "lldb/Host/HostThread.h" #include "lldb/Host/StreamFile.h" @@ -31,6 +33,7 @@ #include "lldb/Utility/Diagnostics.h" #include "lldb/Utility/FileSpec.h" #include "lldb/Utility/Status.h" +#include "lldb/Utility/StructuredData.h" #include "lldb/Utility/UserID.h" #include "lldb/lldb-defines.h" #include "lldb/lldb-enumerations.h" @@ -46,6 +49,7 @@ #include "llvm/Support/DynamicLibrary.h" #include "llvm/Support/FormatVariadic.h" #include "llvm/Support/Threading.h" +#include "llvm/Telemetry/Telemetry.h" #include #include @@ -149,6 +153,10 @@ class Debugger : public std::enable_shared_from_this, repro::DataRecorder *GetInputRecorder(); + TelemetryManager *GetTelemetryManager() { return m_telemetry_manager.get(); } + + void SendClientTelemetry(const lldb_private::StructuredDataImpl &entry); + Status SetInputString(const char *data); void SetInputFile(lldb::FileSP file); @@ -759,6 +767,8 @@ class Debugger : public std::enable_shared_from_this, eBroadcastBitEventThreadIsListening = (1 << 0), }; + std::unique_ptr m_telemetry_manager; + private: // Use Debugger::CreateInstance() to get a shared pointer to a new debugger // object diff --git a/lldb/include/lldb/Core/PluginManager.h b/lldb/include/lldb/Core/PluginManager.h index e4e0c3eea67f8..f17104cfdc796 100644 --- a/lldb/include/lldb/Core/PluginManager.h +++ b/lldb/include/lldb/Core/PluginManager.h @@ -379,6 +379,15 @@ class PluginManager { const UUID *uuid, const ArchSpec *arch); + // TelemetryVendor + static bool RegisterPlugin(llvm::StringRef name, llvm::StringRef description, + TelemetryVendorCreateInstance create_callback); + + static bool UnregisterPlugin(TelemetryVendorCreateInstance create_callback); + + static TelemetryVendorCreateInstance + GetTelemetryVendorCreateCallbackAtIndex(uint32_t idx); + // Trace static bool RegisterPlugin( llvm::StringRef name, llvm::StringRef description, diff --git a/lldb/include/lldb/Core/Telemetry.h b/lldb/include/lldb/Core/Telemetry.h new file mode 100644 index 0000000000000..4d7dfe517ce39 --- /dev/null +++ b/lldb/include/lldb/Core/Telemetry.h @@ -0,0 +1,308 @@ +//===-- Telemetry.h ----------------------------------------------*- C++ +//-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLDB_CORE_TELEMETRY_H +#define LLDB_CORE_TELEMETRY_H + +#include +#include +#include +#include +#include +#include +#include + +#include "lldb/Core/StructuredDataImpl.h" +#include "lldb/Interpreter/CommandReturnObject.h" +#include "lldb/Utility/StructuredData.h" +#include "lldb/lldb-forward.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/JSON.h" +#include "llvm/Telemetry/Telemetry.h" + +namespace lldb_private { + +using llvm::telemetry::Destination; +using llvm::telemetry::KindType; +using llvm::telemetry::Serializer; +using llvm::telemetry::TelemetryInfo; + +struct LldbEntryKind : public ::llvm::telemetry::EntryKind { + static const KindType BaseInfo = 0b11000; + static const KindType DebuggerInfo = 0b11001; + static const KindType TargetInfo = 0b11010; + static const KindType ClientInfo = 0b11100; + static const KindType CommandInfo = 0b11101; + static const KindType MiscInfo = 0b11110; +}; + +/// Defines a convenient type for timestamp of various events. +/// This is used by the EventStats below. +using SteadyTimePoint = std::chrono::time_point; + +/// Various time (and possibly memory) statistics of an event. +struct EventStats { + // REQUIRED: Start time of an event + SteadyTimePoint start; + // OPTIONAL: End time of an event - may be empty if not meaningful. + std::optional end; + // TBD: could add some memory stats here too? + + EventStats() = default; + EventStats(SteadyTimePoint start) : start(start) {} + EventStats(SteadyTimePoint start, SteadyTimePoint end) + : start(start), end(end) {} +}; + +/// Describes the exit signal of an event. +struct ExitDescription { + int exit_code; + std::string description; +}; + +struct LldbBaseTelemetryInfo : public TelemetryInfo { + EventStats stats; + + // For dyn_cast, isa, etc operations. + KindType getKind() const override { return LldbEntryKind::BaseInfo; } + + static bool classof(const TelemetryInfo *t) { + // Subclasses of this is also acceptable. + return (t->getKind() & LldbEntryKind::BaseInfo) == LldbEntryKind::BaseInfo; + } + + void serialize(Serializer &serializer) const override; +}; + +struct DebuggerTelemetryInfo : public LldbBaseTelemetryInfo { + std::string username; + std::string lldb_git_sha; + std::string lldb_path; + std::string cwd; + + std::optional exit_desc; + DebuggerTelemetryInfo() = default; + + // Provide a copy ctor because we may need to make a copy before + // sanitizing the data. + // (The sanitization might differ between different Destination classes). + DebuggerTelemetryInfo(const DebuggerTelemetryInfo &other) { + username = other.username; + lldb_git_sha = other.lldb_git_sha; + lldb_path = other.lldb_path; + cwd = other.cwd; + }; + + KindType getKind() const override { return LldbEntryKind::DebuggerInfo; } + + static bool classof(const TelemetryInfo *T) { + if (T == nullptr) + return false; + return T->getKind() == LldbEntryKind::DebuggerInfo; + } + + void serialize(Serializer &serializer) const override; +}; + +struct TargetTelemetryInfo : public LldbBaseTelemetryInfo { + lldb::ModuleSP exec_mod; + Target *target_ptr; + + // The same as the executable-module's UUID. + std::string target_uuid; + std::string file_format; + + std::string binary_path; + size_t binary_size; + + std::optional exit_desc; + TargetTelemetryInfo() = default; + + TargetTelemetryInfo(const TargetTelemetryInfo &other) { + exec_mod = other.exec_mod; + target_uuid = other.target_uuid; + file_format = other.file_format; + binary_path = other.binary_path; + binary_size = other.binary_size; + exit_desc = other.exit_desc; + } + + KindType getKind() const override { return LldbEntryKind::TargetInfo; } + + static bool classof(const TelemetryInfo *T) { + if (T == nullptr) + return false; + return T->getKind() == LldbEntryKind::TargetInfo; + } + + void serialize(Serializer &serializer) const override; +}; + +// Entry from client (eg., SB-API) +struct ClientTelemetryInfo : public LldbBaseTelemetryInfo { + std::string request_name; + std::string error_msg; + + ClientTelemetryInfo() = default; + + ClientTelemetryInfo(const ClientTelemetryInfo &other) { + request_name = other.request_name; + error_msg = other.error_msg; + } + + KindType getKind() const override { return LldbEntryKind::ClientInfo; } + + static bool classof(const TelemetryInfo *T) { + if (T == nullptr) + return false; + return T->getKind() == LldbEntryKind::ClientInfo; + } + + void serialize(Serializer &serializer) const override; +}; + +struct CommandTelemetryInfo : public LldbBaseTelemetryInfo { + Target *target_ptr; + CommandReturnObject *result; + + // If the command is/can be associated with a target entry, + // this field contains that target's UUID. + // otherwise. + std::string target_uuid; + std::string command_uuid; + + // Eg., "breakpoint set" + std::string command_name; + + // !!NOTE!!: The following fields may be omitted due to PII risk. + // (Configurable via the telemery::Config struct) + std::string original_command; + std::string args; + + std::optional exit_desc; + lldb::ReturnStatus ret_status; + + CommandTelemetryInfo() = default; + + CommandTelemetryInfo(const CommandTelemetryInfo &other) { + target_uuid = other.target_uuid; + command_uuid = other.command_uuid; + command_name = other.command_name; + original_command = other.original_command; + args = other.args; + exit_desc = other.exit_desc; + ret_status = other.ret_status; + } + + KindType getKind() const override { return LldbEntryKind::CommandInfo; } + + static bool classof(const TelemetryInfo *T) { + if (T == nullptr) + return false; + return T->getKind() == LldbEntryKind::CommandInfo; + } + + void serialize(Serializer &serializer) const override; +}; + +/// The "catch-all" entry to store a set of custom/non-standard +/// data. +struct MiscTelemetryInfo : public LldbBaseTelemetryInfo { + /// If the event is/can be associated with a target entry, + /// this field contains that target's UUID. + /// otherwise. + std::string target_uuid; + + /// Set of key-value pairs for any optional (or impl-specific) data + std::map meta_data; + + MiscTelemetryInfo() = default; + + MiscTelemetryInfo(const MiscTelemetryInfo &other) { + target_uuid = other.target_uuid; + meta_data = other.meta_data; + } + + KindType getKind() const override { return LldbEntryKind::MiscInfo; } + + static bool classof(const TelemetryInfo *T) { + if (T == nullptr) + return false; + return T->getKind() == LldbEntryKind::MiscInfo; + } + + void serialize(Serializer &serializer) const override; +}; + +/// The base Telemetry manager instance in LLDB +/// This class declares additional instrumentation points +/// applicable to LLDB. +class TelemetryManager : public llvm::telemetry::Manager { +public: + /// Creates an instance of TelemetryManager. + /// This uses the plugin registry to find an instance: + /// - If a vendor supplies a implementation, it will use it. + /// - If not, it will either return a no-op instance or a basic + /// implementation for testing. + /// + /// See also lldb_private::TelemetryVendor. + static std::unique_ptr + CreateInstance(std::unique_ptr config, + Debugger *debugger); + + /// To be invoked upon LLDB startup. + virtual void LogStartup(DebuggerTelemetryInfo *entry); + + /// To be invoked upon LLDB exit. + virtual void LogExit(DebuggerTelemetryInfo *entry); + + /// To be invoked upon loading the main executable module. + /// We log in a fire-n-forget fashion so that if the load + /// crashes, we don't lose the entry. + virtual void LogMainExecutableLoadStart(TargetTelemetryInfo *entry); + virtual void LogMainExecutableLoadEnd(TargetTelemetryInfo *entry); + + /// To be invoked upon process exit. + virtual void LogProcessExit(TargetTelemetryInfo *entry); + + /// Invoked for each command + /// We log in a fire-n-forget fashion so that if the command execution + /// crashes, we don't lose the entry. + virtual void LogCommandStart(CommandTelemetryInfo *entry); + virtual void LogCommandEnd(CommandTelemetryInfo *entry); + + /// For client (eg., SB API) to send telemetry entries. + virtual void + LogClientTelemetry(const lldb_private::StructuredDataImpl &entry); + + virtual std::string GetNextUUID() { + return std::to_string(uuid_seed.fetch_add(1)); + } + + llvm::Error dispatch(TelemetryInfo *entry) override; + void addDestination(std::unique_ptr destination) override; + +protected: + TelemetryManager(std::unique_ptr config, + Debugger *debugger); + TelemetryManager() = default; + virtual void CollectMiscBuildInfo(); + +private: + std::atomic uuid_seed = 0; + std::unique_ptr m_config; + Debugger *m_debugger; + const std::string m_session_uuid; + std::vector> m_destinations; +}; + +} // namespace lldb_private +#endif // LLDB_CORE_TELEMETRY_H diff --git a/lldb/include/lldb/Core/TelemetryVendor.h b/lldb/include/lldb/Core/TelemetryVendor.h new file mode 100644 index 0000000000000..e484dca07296b --- /dev/null +++ b/lldb/include/lldb/Core/TelemetryVendor.h @@ -0,0 +1,44 @@ +//===-- TelemetryVendor.h ---------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLDB_CORE_TELEMETRYVENDOR_H +#define LLDB_CORE_TELEMETRYVENDOR_H + +#include "lldb/Core/PluginInterface.h" +#include "lldb/Core/Telemetry.h" +#include "llvm/Telemetry/Telemetry.h" + +#include + +namespace lldb_private { + +class TelemetryVendor : public PluginInterface { +public: + static TelemetryVendor *FindPlugin(); + + TelemetryVendor() = default; + + llvm::StringRef GetPluginName() override; + + std::unique_ptr GetTelemetryConfig(); + + // Creates a TelemetryManager instance. + // Vendor plugins can override this to create customized instance as needed. + virtual std::unique_ptr + CreateTelemetryManager(lldb_private::Debugger *debugger); + +protected: + // Returns a vendor-specific config which may or may not be the same as the + // given "default_config". Downstream implementation can define their + // configugrations in addition to OR overriding the default option. + virtual std::unique_ptr GetVendorSpecificConfig( + std::unique_ptr default_config); +}; + +} // namespace lldb_private +#endif // LLDB_CORE_TELEMETRYVENDOR_H diff --git a/lldb/include/lldb/lldb-enumerations.h b/lldb/include/lldb/lldb-enumerations.h index 938f6e3abe8f2..8015f42c5ffc8 100644 --- a/lldb/include/lldb/lldb-enumerations.h +++ b/lldb/include/lldb/lldb-enumerations.h @@ -257,8 +257,8 @@ enum StopReason { }; /// Command Return Status Types. -enum ReturnStatus { - eReturnStatusInvalid, +enum ReturnStatus : int { + eReturnStatusInvalid = 0, eReturnStatusSuccessFinishNoResult, eReturnStatusSuccessFinishResult, eReturnStatusSuccessContinuingNoResult, diff --git a/lldb/include/lldb/lldb-forward.h b/lldb/include/lldb/lldb-forward.h index d09edeeccaff1..b9bfd8f6df29b 100644 --- a/lldb/include/lldb/lldb-forward.h +++ b/lldb/include/lldb/lldb-forward.h @@ -236,6 +236,7 @@ class SystemRuntime; class Target; class TargetList; class TargetProperties; +class TelemetryVendor; class Thread; class ThreadCollection; class ThreadList; diff --git a/lldb/include/lldb/lldb-private-interfaces.h b/lldb/include/lldb/lldb-private-interfaces.h index 5bac5cd3e86b5..a30125813e688 100644 --- a/lldb/include/lldb/lldb-private-interfaces.h +++ b/lldb/include/lldb/lldb-private-interfaces.h @@ -128,6 +128,7 @@ typedef bool (*ScriptedInterfaceCreateInstance)(lldb::ScriptLanguage language, ScriptedInterfaceUsages usages); typedef int (*ComparisonFunction)(const void *, const void *); typedef void (*DebuggerInitializeCallback)(Debugger &debugger); +typedef TelemetryVendor *(*TelemetryVendorCreateInstance)(); /// Trace /// \{ typedef llvm::Expected (*TraceCreateInstanceFromBundle)( diff --git a/lldb/source/API/SBDebugger.cpp b/lldb/source/API/SBDebugger.cpp index 6b72994fc96af..0d6df7907440b 100644 --- a/lldb/source/API/SBDebugger.cpp +++ b/lldb/source/API/SBDebugger.cpp @@ -973,6 +973,16 @@ SBTarget SBDebugger::GetDummyTarget() { return sb_target; } +void SBDebugger::SendTelemetry(const lldb::SBStructuredData &entry) { + if (lldb_private::Debugger *debugger = this->get()) { + debugger->SendClientTelemetry(*(entry.m_impl_up.get())); + } else { + Log *log = GetLog(LLDBLog::API); + LLDB_LOGF(log, + "Could not send telemetry from SBDebugger - debugger was null."); + } +} + bool SBDebugger::DeleteTarget(lldb::SBTarget &target) { LLDB_INSTRUMENT_VA(this, target); diff --git a/lldb/source/Core/CMakeLists.txt b/lldb/source/Core/CMakeLists.txt index dbc620b91b1ed..0632c42e78c65 100644 --- a/lldb/source/Core/CMakeLists.txt +++ b/lldb/source/Core/CMakeLists.txt @@ -51,6 +51,8 @@ add_lldb_library(lldbCore Section.cpp SourceLocationSpec.cpp SourceManager.cpp + Telemetry.cpp + TelemetryVendor.cpp StreamAsynchronousIO.cpp ThreadedCommunication.cpp UserSettingsController.cpp @@ -94,6 +96,7 @@ add_lldb_library(lldbCore Support Demangle TargetParser + Telemetry ) add_dependencies(lldbCore diff --git a/lldb/source/Core/Debugger.cpp b/lldb/source/Core/Debugger.cpp index 9bdc5a3949751..9a56dbdb048da 100644 --- a/lldb/source/Core/Debugger.cpp +++ b/lldb/source/Core/Debugger.cpp @@ -17,6 +17,7 @@ #include "lldb/Core/PluginManager.h" #include "lldb/Core/Progress.h" #include "lldb/Core/StreamAsynchronousIO.h" +#include "lldb/Core/TelemetryVendor.h" #include "lldb/DataFormatters/DataVisualization.h" #include "lldb/Expression/REPL.h" #include "lldb/Host/File.h" @@ -69,6 +70,7 @@ #include "llvm/Support/Threading.h" #include "llvm/Support/raw_ostream.h" +#include #include #include #include @@ -733,12 +735,17 @@ void Debugger::InstanceInitialize() { DebuggerSP Debugger::CreateInstance(lldb::LogOutputCallback log_callback, void *baton) { + SteadyTimePoint start_time = std::chrono::steady_clock::now(); DebuggerSP debugger_sp(new Debugger(log_callback, baton)); if (g_debugger_list_ptr && g_debugger_list_mutex_ptr) { std::lock_guard guard(*g_debugger_list_mutex_ptr); g_debugger_list_ptr->push_back(debugger_sp); } debugger_sp->InstanceInitialize(); + DebuggerTelemetryInfo entry; + entry.lldb_path = HostInfo::GetProgramFileSpec().GetPath(); + entry.stats = {start_time, std::chrono::steady_clock::now()}; + debugger_sp->m_telemetry_manager->LogStartup(&entry); return debugger_sp; } @@ -860,7 +867,9 @@ Debugger::Debugger(lldb::LogOutputCallback log_callback, void *baton) m_sync_broadcaster(nullptr, "lldb.debugger.sync"), m_broadcaster(m_broadcaster_manager_sp, GetStaticBroadcasterClass().str()), - m_forward_listener_sp(), m_clear_once() { + m_forward_listener_sp(), m_clear_once(), + m_telemetry_manager( + TelemetryVendor::FindPlugin()->CreateTelemetryManager(this)) { // Initialize the debugger properties as early as possible as other parts of // LLDB will start querying them during construction. m_collection_sp->Initialize(g_debugger_properties); @@ -952,6 +961,7 @@ void Debugger::Clear() { // static void Debugger::Destroy(lldb::DebuggerSP &debugger_sp); // static void Debugger::Terminate(); llvm::call_once(m_clear_once, [this]() { + SteadyTimePoint quit_start_time = std::chrono::steady_clock::now(); ClearIOHandlers(); StopIOHandlerThread(); StopEventHandlerThread(); @@ -974,6 +984,13 @@ void Debugger::Clear() { if (Diagnostics::Enabled()) Diagnostics::Instance().RemoveCallback(m_diagnostics_callback_id); + + // Log the "quit" event (including stats on how long the teardown took) + // TBD: We *may* have to send off the log BEFORE the ClearIOHanders()? + DebuggerTelemetryInfo entry; + entry.stats = {quit_start_time, std::chrono::steady_clock::now()}; + entry.lldb_path = HostInfo::GetProgramFileSpec().GetPath(); + m_telemetry_manager->LogExit(&entry); }); } @@ -2239,3 +2256,8 @@ llvm::ThreadPoolInterface &Debugger::GetThreadPool() { "Debugger::GetThreadPool called before Debugger::Initialize"); return *g_thread_pool; } + +void Debugger::SendClientTelemetry( + const lldb_private::StructuredDataImpl &entry) { + m_telemetry_manager->LogClientTelemetry(entry); +} diff --git a/lldb/source/Core/PluginManager.cpp b/lldb/source/Core/PluginManager.cpp index a5219025495a9..d7acab7dbcdf6 100644 --- a/lldb/source/Core/PluginManager.cpp +++ b/lldb/source/Core/PluginManager.cpp @@ -1224,6 +1224,33 @@ FileSpec PluginManager::FindSymbolFileInBundle(const FileSpec &symfile_bundle, return {}; } +#pragma mark TelemetryVendor + +typedef PluginInstance TelemetryVendorInstance; +typedef PluginInstances TelemetryVendorInstances; + +static TelemetryVendorInstances &GetTelemetryVendorInstances() { + static TelemetryVendorInstances g_instances; + return g_instances; +} + +bool PluginManager::RegisterPlugin( + llvm::StringRef name, llvm::StringRef description, + TelemetryVendorCreateInstance create_callback) { + return GetTelemetryVendorInstances().RegisterPlugin(name, description, + create_callback); +} + +bool PluginManager::UnregisterPlugin( + TelemetryVendorCreateInstance create_callback) { + return GetTelemetryVendorInstances().UnregisterPlugin(create_callback); +} + +TelemetryVendorCreateInstance +PluginManager::GetTelemetryVendorCreateCallbackAtIndex(uint32_t idx) { + return GetTelemetryVendorInstances().GetCallbackAtIndex(idx); +} + #pragma mark Trace struct TraceInstance diff --git a/lldb/source/Core/Telemetry.cpp b/lldb/source/Core/Telemetry.cpp new file mode 100644 index 0000000000000..e37890aecb867 --- /dev/null +++ b/lldb/source/Core/Telemetry.cpp @@ -0,0 +1,333 @@ + +//===-- Telemetry.cpp -----------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#include "lldb/Core/Telemetry.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "lldb/API/SBDebugger.h" +#include "lldb/API/SBProcess.h" +#include "lldb/Core/Debugger.h" +#include "lldb/Core/Module.h" +#include "lldb/Core/TelemetryVendor.h" +#include "lldb/Host/FileSystem.h" +#include "lldb/Host/HostInfo.h" +#include "lldb/Interpreter/CommandInterpreter.h" +#include "lldb/Target/Process.h" +#include "lldb/Target/Statistics.h" +#include "lldb/Utility/ConstString.h" +#include "lldb/Utility/FileSpec.h" +#include "lldb/Utility/LLDBLog.h" +#include "lldb/Utility/UUID.h" +#include "lldb/Version/Version.h" +#include "lldb/lldb-enumerations.h" +#include "lldb/lldb-forward.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/JSON.h" +#include "llvm/Support/LineIterator.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/RandomNumberGenerator.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Telemetry/Telemetry.h" + +namespace lldb_private { + +using ::llvm::Error; +using ::llvm::telemetry::Destination; +using ::llvm::telemetry::TelemetryInfo; + +static unsigned long long ToNanosec(const SteadyTimePoint Point) { + return nanoseconds(Point.value().time_since_epoch()).count(); +} + +void LldbBaseTelemetryInfo::serialize(Serializer &serializer) const { + serializer.write("EntryKind", getKind()); + serializer.write("SessionId", SessionId); +} + +void DebuggerTelemetryInfo::serialize(Serializer &serializer) const { + LldbBaseTelemetryInfo::serialize(serializer); + serializer.write("username", username); + serializer.write("lldb_path", lldb_path); + serializer.write("cwd", cwd); + serializer.write("start", ToNanosec(stats.start)); + if (stats.end.has_value()) + serializer.write("end", ToNanosec(stats.end.value())); +} + +void ClientTelemetryInfo::serialize(Serializer &serializer) const { + LldbBaseTelemetryInfo::serialize(serializer); + serializer.write("request_name", request_name); + serializer.write("error_msg", error_msg); + serializer.write("start", ToNanosec(stats.start)); + if (stats.end.has_value()) + serializer.write("end", ToNanosec(stats.end.value())); +} + +void TargetTelemetryInfo::serialize(Serializer &serializer) const { + LldbBaseTelemetryInfo::serialize(serializer); + serializer.write("target_uuid", target_uuid); + serializer.write("binary_path", binary_path); + serializer.write("binary_size", binary_size); +} + +void CommandTelemetryInfo::serialize(Serializer &serializer) const { + LldbBaseTelemetryInfo::serialize(serializer); + serializer.write("target_uuid", target_uuid); + serializer.write("command_uuid", command_uuid); + serializer.write("args", args); + serializer.write("original_command", original_command); + serializer.write("start", ToNanosec(stats.start)); + if (stats.end.has_value()) + serializer.write("end", ToNanosec(stats.end.value())); + + if (exit_desc.has_value()) { + serializer.write("exit_code", exit_desc->exit_code); + serializer.write("exit_msg", exit_desc->description); + serializer.write("return_status", static_cast(ret_status)); + } +} + +void MiscTelemetryInfo::serialize(Serializer &serializer) const { + LldbBaseTelemetryInfo::serialize(serializer); + serializer.write("target_uuid", target_uuid); + write.beginObject("meta_data"); + for (const auto &kv : meta_data) + serializer.write(kv.first, kv.second); + serializer.endObject(); +} + +static std::string MakeUUID(lldb_private::Debugger *debugger) { + std::string ret; + uint8_t random_bytes[16]; + if (auto ec = llvm::getRandomBytes(random_bytes, 16)) { + LLDB_LOG(GetLog(LLDBLog::Object), + "Failed to generate random bytes for UUID: {0}", ec.message()); + // fallback to using timestamp + debugger ID. + ret = std::to_string( + std::chrono::steady_clock::now().time_since_epoch().count()) + + "_" + std::to_string(debugger->GetID()); + } else { + ret = lldb_private::UUID(random_bytes).GetAsString(); + } + + return ret; +} + +TelemetryManager::TelemetryManager( + std::unique_ptr config, + lldb_private::Debugger *debugger) + : m_config(std::move(config)), m_debugger(debugger), + m_session_uuid(MakeUUID(debugger)) {} + +std::unique_ptr TelemetryManager::CreateInstance( + std::unique_ptr config, + lldb_private::Debugger *debugger) { + return std::unique_ptr( + new TelemetryManager(std::move(config), debugger)); +} + +llvm::Error TelemetryManager::dispatch(TelemetryInfo *entry) { + entry->SessionId = m_session_uuid; + + llvm::Error defferedErrs = llvm::Error::success(); + for (auto &destination : m_destinations) { + if (auto err = destination->receiveEntry(entry)) + deferredErrs = llvm::joinErrors(std::move(deferredErrs), std::move(err)); + } + return std::move(deferredErrs); +} + +void TelemetryManager::addDestination( + std::unique_ptr destination) { + m_destinations.push_back(std::move(destination)); +} + +void TelemetryManager::LogStartup(DebuggerTelemetryInfo *entry) { + UserIDResolver &resolver = lldb_private::HostInfo::GetUserIDResolver(); + std::optional opt_username = + resolver.GetUserName(lldb_private::HostInfo::GetUserID()); + if (opt_username) + entry->username = *opt_username; + + entry->lldb_git_sha = + lldb_private::GetVersion(); // TODO: find the real git sha? + + llvm::SmallString<64> cwd; + if (!llvm::sys::fs::current_path(cwd)) { + entry->cwd = cwd.c_str(); + } else { + MiscTelemetryInfo misc_info; + misc_info.meta_data["internal_errors"] = "Cannot determine CWD"; + if (auto er = dispatch(&misc_info)) { + LLDB_LOG(GetLog(LLDBLog::Object), + "Failed to dispatch misc-info from startup"); + } + } + + if (auto er = dispatch(entry)) { + LLDB_LOG(GetLog(LLDBLog::Object), "Failed to dispatch entry from startup"); + } + + // Optional part + CollectMiscBuildInfo(); +} + +void TelemetryManager::LogExit(DebuggerTelemetryInfo *entry) { + if (auto *selected_target = + m_debugger->GetSelectedExecutionContext().GetTargetPtr()) { + if (!selected_target->IsDummyTarget()) { + const lldb::ProcessSP proc = selected_target->GetProcessSP(); + if (proc == nullptr) { + // no process has been launched yet. + entry->exit_desc = {-1, "no process launched."}; + } else { + entry->exit_desc = {proc->GetExitStatus(), ""}; + if (const char *description = proc->GetExitDescription()) + entry->exit_desc->description = std::string(description); + } + } + } + dispatch(entry); +} + +void TelemetryManager::LogProcessExit(TargetTelemetryInfo *entry) { + entry->target_uuid = + entry->target_ptr && !entry->target_ptr->IsDummyTarget() + ? entry->target_ptr->GetExecutableModule()->GetUUID().GetAsString() + : ""; + + dispatch(entry); +} + +void TelemetryManager::CollectMiscBuildInfo() { + // collecting use-case specific data +} + +void TelemetryManager::LogMainExecutableLoadStart(TargetTelemetryInfo *entry) { + entry->binary_path = + entry->exec_mod->GetFileSpec().GetPathAsConstString().GetCString(); + entry->file_format = entry->exec_mod->GetArchitecture().GetArchitectureName(); + entry->target_uuid = entry->exec_mod->GetUUID().GetAsString(); + if (auto err = llvm::sys::fs::file_size( + entry->exec_mod->GetFileSpec().GetPath(), entry->binary_size)) { + // If there was error obtaining it, just reset the size to 0. + // Maybe log the error too? + entry->binary_size = 0; + } + dispatch(entry); +} + +void TelemetryManager::LogMainExecutableLoadEnd(TargetTelemetryInfo *entry) { + lldb::ModuleSP exec_mod = entry->exec_mod; + entry->binary_path = + exec_mod->GetFileSpec().GetPathAsConstString().GetCString(); + entry->file_format = exec_mod->GetArchitecture().GetArchitectureName(); + entry->target_uuid = exec_mod->GetUUID().GetAsString(); + entry->binary_size = exec_mod->GetObjectFile()->GetByteSize(); + + dispatch(entry); + + // Collect some more info, might be useful? + MiscTelemetryInfo misc_info; + misc_info.target_uuid = exec_mod->GetUUID().GetAsString(); + misc_info.meta_data["symtab_index_time"] = + std::to_string(exec_mod->GetSymtabIndexTime().get().count()); + misc_info.meta_data["symtab_parse_time"] = + std::to_string(exec_mod->GetSymtabParseTime().get().count()); + dispatch(&misc_info); +} + +void TelemetryManager::LogClientTelemetry( + const lldb_private::StructuredDataImpl &entry) { + // TODO: pull the dictionary out of entry + ClientTelemetryInfo client_info; + /* + std::optional request_name = entry.getString("request_name"); + if (!request_name.has_value()) { + MiscTelemetryInfo misc_info = MakeBaseEntry(); + misc_info.meta_data["internal_errors"] = + "Cannot determine request name from client entry"; + // TODO: Dump the errornous entry to stderr too? + EmitToDestinations(&misc_info); + return; + } + client_info.request_name = request_name->str(); + + std::optional start_time = entry.getInteger("start_time"); + std::optional end_time = entry.getInteger("end_time"); + + if (!start_time.has_value() || !end_time.has_value()) { + MiscTelemetryInfo misc_info = MakeBaseEntry(); + misc_info.meta_data["internal_errors"] = + "Cannot determine start/end time from client entry"; + EmitToDestinations(&misc_info); + return; + } + + SteadyTimePoint epoch; + client_info.Stats.Start = + epoch + std::chrono::nanoseconds(static_cast(*start_time)); + client_info.Stats.End = + epoch + std::chrono::nanoseconds(static_cast(*end_time)); + + std::optional error_msg = entry.getString("error"); + if (error_msg.has_value()) + client_info.error_msg = error_msg->str(); + */ + + dispatch(&client_info); +} + +void TelemetryManager::LogCommandStart(CommandTelemetryInfo *entry) { + // If we have a target attached to this command, then get the UUID. + if (entry->target_ptr && + entry->target_ptr->GetExecutableModule() != nullptr) { + entry->target_uuid = + entry->target_ptr->GetExecutableModule()->GetUUID().GetAsString(); + } else { + entry->target_uuid = ""; + } + + dispatch(entry); +} + +void TelemetryManager::LogCommandEnd(CommandTelemetryInfo *entry) { + // If we have a target attached to this command, then get the UUID. + if (entry->target_ptr && + entry->target_ptr->GetExecutableModule() != nullptr) { + entry->target_uuid = + entry->target_ptr->GetExecutableModule()->GetUUID().GetAsString(); + } else { + entry->target_uuid = ""; + } + + entry->exit_desc = {entry->result->Succeeded() ? 0 : -1, ""}; + if (llvm::StringRef error_data = entry->result->GetErrorData(); + !error_data.empty()) { + entry->exit_desc->description = error_data.str(); + } + entry->ret_status = entry->result->GetStatus(); + dispatch(entry); +} + +} // namespace lldb_private diff --git a/lldb/source/Core/TelemetryVendor.cpp b/lldb/source/Core/TelemetryVendor.cpp new file mode 100644 index 0000000000000..63bca9352e17e --- /dev/null +++ b/lldb/source/Core/TelemetryVendor.cpp @@ -0,0 +1,275 @@ +//===-- TelemetryVendor.cpp -------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "lldb/Core/TelemetryVendor.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "lldb/API/SBDebugger.h" +#include "lldb/API/SBProcess.h" +#include "lldb/Core/Debugger.h" +#include "lldb/Core/Module.h" +#include "lldb/Core/PluginManager.h" +#include "lldb/Core/Telemetry.h" +#include "lldb/Host/FileSystem.h" +#include "lldb/Host/HostInfo.h" +#include "lldb/Interpreter/CommandInterpreter.h" +#include "lldb/Target/Process.h" +#include "lldb/Target/Statistics.h" +#include "lldb/Utility/ConstString.h" +#include "lldb/Utility/FileSpec.h" +#include "lldb/Utility/LLDBLog.h" +#include "lldb/Utility/UUID.h" +#include "lldb/Version/Version.h" +#include "lldb/lldb-enumerations.h" +#include "lldb/lldb-forward.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/JSON.h" +#include "llvm/Support/LineIterator.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/RandomNumberGenerator.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Telemetry/Telemetry.h" + +using namespace lldb; +using namespace lldb_private; + +namespace { + +using ::llvm::Error; +using ::llvm::StringRef; +using ::llvm::telemetry::Destination; +using ::llvm::telemetry::TelemetryInfo; + +// No-op logger to use when users disable telemetry +class NoOpTelemeter : public TelemetryManager { +public: + static std::unique_ptr CreateInstance(Debugger *debugger) { + return std::unique_ptr(new NoOpTelemeter(debugger)); + } + + NoOpTelemeter(Debugger *debugger) {} + NoOpTelemeter() = default; + ~NoOpTelemeter() = default; + + void addDestination(std::unique_ptr destination) override {} + llvm::Error dispatch(TelemetryInfo *entry) override { + return Error::success(); + } + void LogStartup(DebuggerTelemetryInfo *entry) override {} + void LogExit(DebuggerTelemetryInfo *entry) override {} + void LogMainExecutableLoadStart(TargetTelemetryInfo *entry) override {} + void LogMainExecutableLoadEnd(TargetTelemetryInfo *entry) override {} + void LogProcessExit(TargetTelemetryInfo *entry) override {} + void LogCommandStart(CommandTelemetryInfo *entry) override {} + void LogCommandEnd(CommandTelemetryInfo *entry) override {} + void + LogClientTelemetry(const lldb_private::StructuredDataImpl &entry) override {} + + std::string GetNextUUID() override { return ""; } +}; + +class BasicSerializer : public Serializer { +public: + const std::string &getString() { return Buffer; } + + llvm::Error start() override { + if (started) + return llvm::createStringError("Serializer already in use"); + started = true; + Buffer.clear(); + return Error::success(); + } + + void writeBool(StringRef KeyName, bool Value) override { + writeHelper(KeyName, Value); + } + + void writeInt32(StringRef KeyName, int Value) override { + writeHelper(KeyName, Value); + } + + void writeSizeT(StringRef KeyName, size_t Value) override { + writeHelper(KeyName, Value); + } + void writeString(StringRef KeyName, StringRef Value) override { + assert(started && "serializer not started"); + } + + void + writeKeyValueMap(StringRef KeyName, + const std::map &Value) override { + std::string Inner; + for (auto kv : Value) { + writeHelper(StringRef(kv.first), StringRef(kv.second), &Inner); + } + writeHelper(KeyName, StringRef(Inner)); + } + + llvm::Error finish() override { + if (!started) + return llvm::createStringError("Serializer not currently in use"); + started = false; + return Error::success(); + } + +private: + template + void writeHelper(StringRef Name, T Value, std::string *Buff) { + assert(started && "serializer not started"); + Buff->append((Name + ":" + llvm::Twine(Value) + "\n").str()); + } + + template void writeHelper(StringRef Name, T Value) { + writeHelper(Name, Value, &Buffer); + } + + bool started = false; + std::string Buffer; +}; + +class StreamTelemetryDestination : public Destination { +public: + StreamTelemetryDestination(llvm::raw_ostream &os) : os(os) {} + llvm::Error + receiveEntry(const llvm::telemetry::TelemetryInfo *entry) override { + // Upstream Telemetry should not leak anything other than the + // basic data, unless running in test mode. +#ifdef TEST_TELEMETRY + if (Error err = serializer.start()) { + return err; + } + entry->serialize(serializer); + if (Error err = serializer.finish()) { + return err; + } + os << serializer.getString() << "\n"; +#else + os << "session_uuid: " << entry->SessionId + << "\n"; +#endif + os.flush(); + return llvm::ErrorSuccess(); + } + + llvm::StringLiteral name() const override { return "StreamDestination"; } + +private: + llvm::raw_ostream &os; + BasicSerializer serializer; +}; +} // namespace + +TelemetryVendor *TelemetryVendor::FindPlugin() { + // The default implementation (ie., upstream impl) returns + // the basic instance. + // + // Vendors can provide their plugins as needed. + + std::unique_ptr instance_up; + TelemetryVendorCreateInstance create_callback; + + for (size_t idx = 0; + (create_callback = + PluginManager::GetTelemetryVendorCreateCallbackAtIndex(idx)) != + nullptr; + ++idx) { + instance_up.reset(create_callback()); + + if (instance_up) { + return instance_up.release(); + } + } + + return new TelemetryVendor(); +} + +llvm::StringRef TelemetryVendor::GetPluginName() { + return "DefaultTelemetryVendor"; +} + +static llvm::StringRef ParseValue(llvm::StringRef str, llvm::StringRef label) { + return str.substr(label.size()).trim(); +} + +static bool ParseBoolValue(llvm::StringRef str, llvm::StringRef label) { + if (ParseValue(str, label) == "true") + return true; + return false; +} + +std::unique_ptr TelemetryVendor::GetTelemetryConfig() { + // Telemetry is disabled by default. + bool enable_telemetry = false; + + // Look in the $HOME/.lldb_telemetry_config file to populate the struct + llvm::SmallString<64> init_file; + FileSystem::Instance().GetHomeDirectory(init_file); + llvm::sys::path::append(init_file, ".lldb_telemetry_config"); + FileSystem::Instance().Resolve(init_file); + if (llvm::sys::fs::exists(init_file)) { + auto contents = llvm::MemoryBuffer::getFile(init_file, /*IsText*/ true); + if (contents) { + llvm::line_iterator iter = + llvm::line_iterator(contents->get()->getMemBufferRef()); + for (; !iter.is_at_eof(); ++iter) { + if (iter->starts_with("enable_telemetry:")) { + enable_telemetry = ParseBoolValue(*iter, "enable_telemetry:"); + } + } + } else { + LLDB_LOG(GetLog(LLDBLog::Object), "Error reading config file at {0}", + init_file.c_str()); + } + } + +// Enable Telemetry in upstream config only if we are running tests. +#ifdef TEST_TELEMETRY + enable_telemetry = true; +#endif + + auto config = std::make_unique(enable_telemetry); + + // Now apply any additional vendor config, if available. + // TODO: cache the Config? (given it's not going to change after LLDB starts + // up) However, it's possible we want to supporting restarting the Telemeter + // with new config? + return GetVendorSpecificConfig(std::move(config)); +} + +std::unique_ptr +TelemetryVendor::GetVendorSpecificConfig( + std::unique_ptr default_config) { + return std::move(default_config); +} + +std::unique_ptr +TelemetryVendor::CreateTelemetryManager(Debugger *debugger) { + auto config = GetTelemetryConfig(); + + if (!config->EnableTelemetry) { + return NoOpTelemeter::CreateInstance(debugger); + } + + return TelemetryManager::CreateInstance(std::move(config), debugger); +} diff --git a/lldb/source/Interpreter/CommandInterpreter.cpp b/lldb/source/Interpreter/CommandInterpreter.cpp index b93f47a8a8d5e..518113d354e5b 100644 --- a/lldb/source/Interpreter/CommandInterpreter.cpp +++ b/lldb/source/Interpreter/CommandInterpreter.cpp @@ -1886,8 +1886,42 @@ bool CommandInterpreter::HandleCommand(const char *command_line, LazyBool lazy_add_to_history, CommandReturnObject &result, bool force_repeat_command) { + EventStats start_command_stats(std::chrono::steady_clock::now()); + TelemetryManager *manager = GetDebugger().GetTelemetryManager(); + // Generate a UUID for this command so the logger can match + // the start/end entries correctly. + const std::string command_uuid = manager->GetNextUUID(); + + CommandTelemetryInfo start_entry; + start_entry.stats = start_command_stats; + start_entry.command_uuid = command_uuid; + start_entry.target_ptr = GetExecutionContext().GetTargetPtr(); + + manager->LogCommandStart(&start_entry); + std::string command_string(command_line); std::string original_command_string(command_line); + std::string parsed_command_args; + CommandObject *cmd_obj = nullptr; + + auto log_on_exit = llvm::make_scope_exit([&]() { + EventStats end_command_stats(start_command_stats.start, + std::chrono::steady_clock::now()); + + llvm::StringRef command_name = + cmd_obj ? cmd_obj->GetCommandName() : ""; + // TODO: this is logging the time the command-handler finishes. + // But we may want a finer-grain durations too? + // (ie., the execute_time recorded below?) + CommandTelemetryInfo end_entry; + end_entry.stats = end_command_stats; + end_entry.command_uuid = command_uuid; + end_entry.command_name = command_name.str(); + end_entry.args = parsed_command_args; + end_entry.target_ptr = GetExecutionContext().GetTargetPtr(); + end_entry.result = &result; + manager->LogCommandEnd(&end_entry); + }); Log *log = GetLog(LLDBLog::Commands); llvm::PrettyStackTraceFormat stack_trace("HandleCommand(command = \"%s\")", @@ -1925,9 +1959,9 @@ bool CommandInterpreter::HandleCommand(const char *command_line, bool empty_command = false; bool comment_command = false; - if (command_string.empty()) + if (command_string.empty()) { empty_command = true; - else { + } else { const char *k_space_characters = "\t\n\v\f\r "; size_t non_space = command_string.find_first_not_of(k_space_characters); @@ -1992,7 +2026,7 @@ bool CommandInterpreter::HandleCommand(const char *command_line, // From 1 above, we can determine whether the Execute function wants raw // input or not. - CommandObject *cmd_obj = ResolveCommandImpl(command_string, result); + cmd_obj = ResolveCommandImpl(command_string, result); // We have to preprocess the whole command string for Raw commands, since we // don't know the structure of the command. For parsed commands, we only @@ -2053,30 +2087,29 @@ bool CommandInterpreter::HandleCommand(const char *command_line, if (add_to_history) m_command_history.AppendString(original_command_string); - std::string remainder; const std::size_t actual_cmd_name_len = cmd_obj->GetCommandName().size(); if (actual_cmd_name_len < command_string.length()) - remainder = command_string.substr(actual_cmd_name_len); + parsed_command_args = command_string.substr(actual_cmd_name_len); // Remove any initial spaces - size_t pos = remainder.find_first_not_of(k_white_space); + size_t pos = parsed_command_args.find_first_not_of(k_white_space); if (pos != 0 && pos != std::string::npos) - remainder.erase(0, pos); + parsed_command_args.erase(0, pos); LLDB_LOGF( log, "HandleCommand, command line after removing command name(s): '%s'", - remainder.c_str()); + parsed_command_args.c_str()); // To test whether or not transcript should be saved, `transcript_item` is // used instead of `GetSaveTrasncript()`. This is because the latter will // fail when the command is "settings set interpreter.save-transcript true". if (transcript_item) { transcript_item->AddStringItem("commandName", cmd_obj->GetCommandName()); - transcript_item->AddStringItem("commandArguments", remainder); + transcript_item->AddStringItem("commandArguments", parsed_command_args); } ElapsedTime elapsed(execute_time); - cmd_obj->Execute(remainder.c_str(), result); + cmd_obj->Execute(parsed_command_args.c_str(), result); } LLDB_LOGF(log, "HandleCommand, command %s", diff --git a/lldb/source/Target/Process.cpp b/lldb/source/Target/Process.cpp index aca0897281147..35c683c818374 100644 --- a/lldb/source/Target/Process.cpp +++ b/lldb/source/Target/Process.cpp @@ -22,6 +22,7 @@ #include "lldb/Core/ModuleSpec.h" #include "lldb/Core/PluginManager.h" #include "lldb/Core/Progress.h" +#include "lldb/Core/Telemetry.h" #include "lldb/Expression/DiagnosticManager.h" #include "lldb/Expression/DynamicCheckerFunctions.h" #include "lldb/Expression/UserExpression.h" @@ -74,6 +75,8 @@ #include "lldb/Utility/SelectHelper.h" #include "lldb/Utility/State.h" #include "lldb/Utility/Timer.h" +#include "llvm/Telemetry/Telemetry.h" +#include using namespace lldb; using namespace lldb_private; @@ -1065,6 +1068,8 @@ bool Process::SetExitStatus(int status, llvm::StringRef exit_string) { // Use a mutex to protect setting the exit status. std::lock_guard guard(m_exit_status_mutex); + SteadyTimePoint start_time = std::chrono::steady_clock::now(); + Log *log(GetLog(LLDBLog::State | LLDBLog::Process)); LLDB_LOG(log, "(plugin = {0} status = {1} ({1:x8}), description=\"{2}\")", GetPluginName(), status, exit_string); @@ -1094,6 +1099,12 @@ bool Process::SetExitStatus(int status, llvm::StringRef exit_string) { // Allow subclasses to do some cleanup DidExit(); + TargetTelemetryInfo entry; + entry.stats = {start_time, std::chrono::steady_clock::now()}; + entry.exit_desc = {status, exit_string.str()}; + entry.target_ptr = &GetTarget(); + GetTarget().GetDebugger().GetTelemetryManager()->LogProcessExit(&entry); + return true; } diff --git a/lldb/source/Target/Target.cpp b/lldb/source/Target/Target.cpp index 29e9efb83efeb..3a5a511cb46cc 100644 --- a/lldb/source/Target/Target.cpp +++ b/lldb/source/Target/Target.cpp @@ -24,6 +24,7 @@ #include "lldb/Core/Section.h" #include "lldb/Core/SourceManager.h" #include "lldb/Core/StructuredDataImpl.h" +#include "lldb/Core/Telemetry.h" #include "lldb/Core/ValueObject.h" #include "lldb/Core/ValueObjectConstResult.h" #include "lldb/Expression/DiagnosticManager.h" @@ -68,7 +69,9 @@ #include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/SetVector.h" +#include "llvm/Telemetry/Telemetry.h" +#include #include #include #include @@ -1552,9 +1555,25 @@ void Target::DidExec() { void Target::SetExecutableModule(ModuleSP &executable_sp, LoadDependentFiles load_dependent_files) { + EventStats load_executable_stats(std::chrono::steady_clock::now()); Log *log = GetLog(LLDBLog::Target); ClearModules(false); + if (executable_sp) { + TargetTelemetryInfo load_start; + load_start.stats = load_executable_stats; + load_start.exec_mod = executable_sp; + + m_debugger.GetTelemetryManager()->LogMainExecutableLoadStart(&load_start); + + auto log_on_exit = llvm::make_scope_exit([&]() { + load_executable_stats.end = std::chrono::steady_clock::now(); + TargetTelemetryInfo load_end; + load_end.stats = load_executable_stats; + load_end.exec_mod = executable_sp; + m_debugger.GetTelemetryManager()->LogMainExecutableLoadEnd(&load_end); + }); + } if (executable_sp) { ElapsedTime elapsed(m_stats.GetCreateTime()); LLDB_SCOPED_TIMERF("Target::SetExecutableModule (executable = '%s')", diff --git a/lldb/test/CMakeLists.txt b/lldb/test/CMakeLists.txt index 5ac474736eb63..6d5a76aaba1b9 100644 --- a/lldb/test/CMakeLists.txt +++ b/lldb/test/CMakeLists.txt @@ -3,7 +3,7 @@ # Lit requires a Python3 interpreter, let's be careful and fail early if it's # not present. if (NOT DEFINED Python3_EXECUTABLE) - message(FATAL_ERROR + message(SEND_ERROR "LLDB test suite requires a Python3 interpreter but none " "was found. Please install Python3 or disable tests with " "`LLDB_INCLUDE_TESTS=OFF`.") @@ -12,7 +12,7 @@ endif() if(LLDB_ENFORCE_STRICT_TEST_REQUIREMENTS) message(STATUS "Enforcing strict test requirements for LLDB") # Lit uses psutil to do per-test timeouts. - set(useful_python_modules psutil) + set(useful_python_modules psutil packaging) if(NOT WIN32) # We no longer vendor pexpect and it is not used on Windows. @@ -22,7 +22,7 @@ if(LLDB_ENFORCE_STRICT_TEST_REQUIREMENTS) foreach(module ${useful_python_modules}) lldb_find_python_module(${module}) if (NOT PY_${module}_FOUND) - message(FATAL_ERROR + message(SEND_ERROR "Python module '${module}' not found. Please install it via pip or via " "your operating system's package manager. Alternatively, disable " "strict testing requirements with " @@ -66,10 +66,10 @@ if (LLDB_TEST_OBJC_GNUSTEP) find_package(GNUstepObjC) if (NOT GNUstepObjC_FOUND) if (LLDB_TEST_OBJC_GNUSTEP_DIR) - message(FATAL_ERROR "Failed to find GNUstep libobjc2 in ${LLDB_TEST_OBJC_GNUSTEP_DIR}. " + message(SEND_ERROR "Failed to find GNUstep libobjc2 in ${LLDB_TEST_OBJC_GNUSTEP_DIR}. " "Please check LLDB_TEST_OBJC_GNUSTEP_DIR or turn off LLDB_TEST_OBJC_GNUSTEP.") else() - message(FATAL_ERROR "Failed to find GNUstep libobjc2. " + message(SEND_ERROR "Failed to find GNUstep libobjc2. " "Please set LLDB_TEST_OBJC_GNUSTEP_DIR or turn off LLDB_TEST_OBJC_GNUSTEP.") endif() endif() @@ -108,6 +108,9 @@ endfunction(add_lldb_test_dependency) add_lldb_test_dependency(lldb) add_lldb_test_dependency(lldb-test) +# Enable Telemetry for testing. +target_compile_definitions(lldb PRIVATE -DTEST_TELEMETRY) + # On Darwin, darwin-debug is an hard dependency for the testsuites. if (CMAKE_SYSTEM_NAME MATCHES "Darwin") add_lldb_test_dependency(darwin-debug) @@ -185,7 +188,7 @@ if(TARGET clang) set(LIBCXX_LIBRARY_DIR "${LLDB_TEST_LIBCXX_ROOT_DIR}/lib${LIBCXX_LIBDIR_SUFFIX}") set(LIBCXX_GENERATED_INCLUDE_DIR "${LLDB_TEST_LIBCXX_ROOT_DIR}/include/c++/v1") else() - message(FATAL_ERROR + message(SEND_ERROR "Couldn't find libcxx build in '${LLDB_TEST_LIBCXX_ROOT_DIR}'. To run the " "test-suite for a standalone LLDB build please build libcxx and point " "LLDB_TEST_LIBCXX_ROOT_DIR to it.") @@ -194,7 +197,7 @@ if(TARGET clang) # We require libcxx for the test suite, so if we aren't building it, # provide a helpful error about how to resolve the situation. if(NOT LLDB_HAS_LIBCXX) - message(FATAL_ERROR + message(SEND_ERROR "LLDB test suite requires libc++, but it is currently disabled. " "Please add `libcxx` to `LLVM_ENABLE_RUNTIMES` or disable tests via " "`LLDB_INCLUDE_TESTS=OFF`.") diff --git a/lldb/tools/lldb-dap/DAP.cpp b/lldb/tools/lldb-dap/DAP.cpp index 884a71ff6693f..d6039b48ebcb7 100644 --- a/lldb/tools/lldb-dap/DAP.cpp +++ b/lldb/tools/lldb-dap/DAP.cpp @@ -13,6 +13,7 @@ #include #include "DAP.h" +#include "JSONUtils.h" #include "LLDBUtils.h" #include "lldb/API/SBCommandInterpreter.h" #include "llvm/ADT/StringExtras.h" @@ -680,17 +681,51 @@ PacketStatus DAP::GetNextObject(llvm::json::Object &object) { } bool DAP::HandleObject(const llvm::json::Object &object) { + auto start_time = std::chrono::steady_clock::now(); const auto packet_type = GetString(object, "type"); if (packet_type == "request") { const auto command = GetString(object, "command"); auto handler_pos = request_handlers.find(std::string(command)); + lldb::SBStructuredData telemetry_entry; + + // There does not seem to be a direct way to construct an SBStructuredData. + // So we first create a json::Array object, + // then we serialize it to a string, + // and finally call SBStructuredData::SetFromJSON(string). + // + // TODO: This seems unnecessarily complex. Ideally, we should be able to + // just send a json::Object directly? Does the SB API allow json? + // + llvm::json::Array telemetry_array({ + {"request_name", std::string(command)}, + {"start_time", start_time.time_since_epoch().count()}, + }); + if (handler_pos != request_handlers.end()) { handler_pos->second(object); + auto end_time = std::chrono::steady_clock::now(); + telemetry_array.push_back( + llvm::json::Value{"end_time", end_time.time_since_epoch().count()}); + + llvm::json::Value val(std::move(telemetry_array)); + std::string string_rep = lldb_dap::JSONToString(val); + telemetry_entry.SetFromJSON(string_rep.c_str()); + debugger.SendTelemetry(telemetry_entry); return true; // Success } else { if (log) *log << "error: unhandled command \"" << command.data() << "\"" << std::endl; + auto end_time = std::chrono::steady_clock::now(); + telemetry_array.push_back( + llvm::json::Value{"end_time", end_time.time_since_epoch().count()}); + telemetry_array.push_back(llvm::json::Value{ + "error", llvm::Twine("unhandled-command:" + command).str()}); + + llvm::json::Value val(std::move(telemetry_array)); + std::string string_rep = lldb_dap::JSONToString(val); + telemetry_entry.SetFromJSON(string_rep.c_str()); + debugger.SendTelemetry(telemetry_entry); return false; // Fail } } diff --git a/llvm/docs/Telemetry.rst b/llvm/docs/Telemetry.rst new file mode 100644 index 0000000000000..af976058e66bc --- /dev/null +++ b/llvm/docs/Telemetry.rst @@ -0,0 +1,213 @@ +=========================== +Telemetry framework in LLVM +=========================== + +.. contents:: + :local: + +.. toctree:: + :hidden: + +=========================== +Telemetry framework in LLVM +=========================== + +Objective +========= + +Provides a common framework in LLVM for collecting various usage and performance +metrics. +It is located at `llvm/Telemetry/Telemetry.h` + +Characteristics +--------------- +* Configurable and extensible by: + + * Tools: any tool that wants to use Telemetry can extend and customize it. + * Vendors: Toolchain vendors can also provide custom implementation of the + library, which could either override or extend the given tool's upstream + implementation, to best fit their organization's usage and privacy models. + * End users of such tool can also configure Telemetry(as allowed by their + vendor). + + +Important notes +---------------- + +* There is no concrete implementation of a Telemetry library in upstream LLVM. + We only provide the abstract API here. Any tool that wants telemetry will + implement one. + + The rationale for this is that, all the tools in llvm are very different in + what they care about(what/where/when to instrument data). Hence, it might not + be practical to have a single implementation. + However, in the future, if we see enough common pattern, we can extract them + into a shared place. This is TBD - contributions are welcomed. + +* No implementation of Telemetry in upstream LLVM shall store any of the + collected data due to privacy and security reasons: + + * Different organizations have different privacy models: + + * Which data is sensitive, which is not? + * Whether it is acceptable for instrumented data to be stored anywhere? + (to a local file, what not?) + + * Data ownership and data collection consents are hard to accommodate from + LLVM developers' point of view: + + * Eg., data collected by Telemetry is not neccessarily owned by the user + of an LLVM tool with Telemetry enabled, hence the user's consent to data + collection is not meaningful. On the other hand, LLVM developers have no + reasonable ways to request consent from the "real" owners. + + +High-level design +================= + +Key components +-------------- + +The framework is consisted of four important classes: + +* `llvm::telemetry::Telemeter`: The class responsible for collecting and + transmitting telemetry data. This is the main point of interaction between the + framework and any tool that wants to enable telemery. +* `llvm::telemetry::TelemetryInfo`: Data courier +* `llvm::telemetry::Destination`: Data sink to which the Telemetry framework + sends data. + Its implementation is transparent to the framework. + It is up to the vendor to decide which pieces of data to forward and where + to forward them to their final storage. +* `llvm::telemetry::Config`: Configurations on the `Telemeter`. + +.. image:: llvm_telemetry_design.png + +How to implement and interact with the API +------------------------------------------ + +To use Telemetry in your tool, you need to provide a concrete implementation of the `Telemeter` class and `Destination`. + +1) Define a custom `Telemeter` and `Destination` + +.. code-block:: c++ + // This destiantion just prints the given entry to a stdout. + // In "real life", this would be where you forward the data to your + // custom data storage. + class MyStdoutDestination : public llvm::telemetry::Destiantion { + public: + Error emitEntry(const TelemetryInfo* Entry) override { + return sendToBlackBox(Entry); + + } + + private: + Error sendToBlackBox(const TelemetryInfo* Entry) { + // This could send the data anywhere. + // But we're simply sending it to stdout for the example. + llvm::outs() << entryToString(Entry) << "\n"; + return llvm::success(); + } + + std::string entryToString(const TelemetryInfo* Entry) { + // make a string-representation of the given entry. + } + }; + + // This defines a custom TelemetryInfo that has an addition Msg field. + struct MyTelemetryInfo : public llvm::telemetry::TelemetryInfo { + std::string Msg; + + json::Object serializeToJson() const { + json::Object Ret = TelemeteryInfo::serializeToJson(); + Ret.emplace_back("MyMsg", Msg); + return std::move(Ret); + } + + // TODO: implement getKind() and classof() to support dyn_cast operations. + }; + + class MyTelemeter : public llvm::telemery::Telemeter { + public: + static std::unique_ptr createInstatnce(llvm::telemetry::Config* config) { + // If Telemetry is not enabled, then just return null; + if (!config->EnableTelemetry) return nullptr; + + std::make_unique(); + } + MyTelemeter() = default; + + void logStartup(llvm::StringRef ToolName, TelemetryInfo* Entry) override { + if (MyTelemetryInfo* M = dyn_cast(Entry)) { + M->Msg = "Starting up tool with name: " + ToolName; + emitToAllDestinations(M); + } else { + emitToAllDestinations(Entry); + } + } + + void logExit(llvm::StringRef ToolName, TelemetryInfo* Entry) override { + if (MyTelemetryInfo* M = dyn_cast(Entry)) { + M->Msg = "Exitting tool with name: " + ToolName; + emitToAllDestinations(M); + } else { + emitToAllDestinations(Entry); + } + } + + void addDestination(Destination* dest) override { + destinations.push_back(dest); + } + + // You can also define additional instrumentation points.) + void logAdditionalPoint(TelemetryInfo* Entry) { + // .... code here + } + private: + void emitToAllDestinations(const TelemetryInfo* Entry) { + // Note: could do this in paralle, if needed. + for (Destination* Dest : Destinations) + Dest->emitEntry(Entry); + } + std::vector Destinations; + }; + +2) Use the library in your tool. + +Logging the tool init-process: + +.. code-block:: c++ + + // At tool's init code + auto StartTime = std::chrono::time_point::now(); + llvm::telemetry::Config MyConfig = makeConfig(); // build up the appropriate Config struct here. + auto Telemeter = MyTelemeter::createInstance(&MyConfig); + std::string CurrentSessionId = ...; // Make some unique ID corresponding to the current session here. + + // Any other tool's init code can go here + // ... + + // Finally, take a snapshot of the time now so we know how long it took the + // init process to finish + auto EndTime = std::chrono::time_point::now(); + MyTelemetryInfo Entry; + Entry.SessionId = CurrentSessionId ; // Assign some unique ID here. + Entry.Stats = {StartTime, EndTime}; + Telemeter->logStartup("MyTool", &Entry); + +Similar code can be used for logging the tool's exit. + +Additionall, at any other point in the tool's lifetime, it can also log telemetry: + +.. code-block:: c++ + + // At some execution point: + auto StartTime = std::chrono::time_point::now(); + + // ... other events happening here + + auto EndTime = std::chrono::time_point::now(); + MyTelemetryInfo Entry; + Entry.SessionId = CurrentSessionId ; // Assign some unique ID here. + Entry.Stats = {StartTime, EndTime}; + Telemeter->logAdditionalPoint(&Entry); diff --git a/llvm/docs/UserGuides.rst b/llvm/docs/UserGuides.rst index 86101ffbd9ca5..feb07bcf0a7ef 100644 --- a/llvm/docs/UserGuides.rst +++ b/llvm/docs/UserGuides.rst @@ -72,6 +72,7 @@ intermediate LLVM representation. SupportLibrary TableGen/index TableGenFundamentals + Telemetry Vectorizers WritingAnLLVMPass WritingAnLLVMNewPMPass @@ -293,3 +294,6 @@ Additional Topics :doc:`Sandbox IR ` This document describes the design and usage of Sandbox IR, a transactional layer over LLVM IR. + +:doc:`Telemetry` + This document describes the Telemetry framework in LLVM. diff --git a/llvm/docs/llvm_telemetry_design.png b/llvm/docs/llvm_telemetry_design.png new file mode 100644 index 0000000000000..0ccb75cf8e30b Binary files /dev/null and b/llvm/docs/llvm_telemetry_design.png differ diff --git a/llvm/include/llvm/Telemetry/Telemetry.h b/llvm/include/llvm/Telemetry/Telemetry.h new file mode 100644 index 0000000000000..7e774e856e0d8 --- /dev/null +++ b/llvm/include/llvm/Telemetry/Telemetry.h @@ -0,0 +1,132 @@ +//===- llvm/Telemetry/Telemetry.h - Telemetry -------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file provides the basic framework for Telemetry +/// Refer to its documentation at llvm/docs/Telemetry.rst for more details. +//===---------------------------------------------------------------------===// + +#ifndef LLVM_TELEMETRY_TELEMETRY_H +#define LLVM_TELEMETRY_TELEMETRY_H + +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/JSON.h" +#include +#include +#include + +namespace llvm { +namespace telemetry { + +class Serializer { +public: + virtual Error init() = 0; + virtual void write(StringRef KeyName, bool Value) = 0; + virtual void write(StringRef KeyName, int Value) = 0; + virtual void write(StringRef KeyName, unsigned long long Value) = 0; + virtual void write(StringRef KeyName, StringRef Value) = 0; + virtual void beginObject(StringRef KeyName) = 0; + virtual void endObject() = 0; + virtual Error finalize() = 0; +}; + +/// Configuration for the Telemeter class. +/// This stores configurations from both users and vendors and is passed +/// to the Telemeter upon construction. (Any changes to the config after +/// the Telemeter's construction will not have any effect on it). +/// +/// This struct can be extended as needed to add additional configuration +/// points specific to a vendor's implementation. +struct Config { + // If true, telemetry will be enabled. + const bool EnableTelemetry; + Config(bool E) : EnableTelemetry(E) {} + + virtual std::string makeSessionId() { return "0"; } +}; + +/// For isa, dyn_cast, etc operations on TelemetryInfo. +typedef unsigned KindType; +/// This struct is used by TelemetryInfo to support isa<>, dyn_cast<> +/// operations. +/// It is defined as a struct (rather than an enum) because it is +/// expected to be extended by subclasses which may have +/// additional TelemetryInfo types defined to describe different events. +struct EntryKind { + static const KindType Base = 0; +}; + +/// TelemetryInfo is the data courier, used to move instrumented data +/// from the tool being monitored to the Telemetry framework. +/// +/// This base class contains only the basic set of telemetry data. +/// Downstream implementations can define more subclasses with +/// additional fields to describe different events and concepts. +/// +/// For example, The LLDB debugger can define a DebugCommandInfo subclass +/// which has additional fields about the debug-command being instrumented, +/// such as `CommandArguments` or `CommandName`. +struct TelemetryInfo { + // This represents a unique-id, conventionally corresponding to + // a tool's session - i.e., every time the tool starts until it exits. + // + // Note: a tool could have multiple sessions running at once, in which + // case, these shall be multiple sets of TelemetryInfo with multiple unique + // ids. + // + // Different usages can assign different types of IDs to this field. + std::string SessionId; + + TelemetryInfo() = default; + virtual ~TelemetryInfo() = default; + + virtual void serialize(Serializer &serializer) const; + + // For isa, dyn_cast, etc, operations. + virtual KindType getKind() const { return EntryKind::Base; } + static bool classof(const TelemetryInfo *T) { + if (T == nullptr) + return false; + return T->getKind() == EntryKind::Base; + } +}; + +/// This class presents a data sink to which the Telemetry framework +/// sends data. +/// +/// Its implementation is transparent to the framework. +/// It is up to the vendor to decide which pieces of data to forward +/// and where to forward them. +class Destination { +public: + virtual ~Destination() = default; + virtual Error receiveEntry(const TelemetryInfo *Entry) = 0; + virtual llvm::StringLiteral name() const = 0; +}; + +/// This class is the main interaction point between any LLVM tool +/// and this framework. +/// It is responsible for collecting telemetry data from the tool being +/// monitored and transmitting the data elsewhere. +class Manager { +public: + // Dispatch Telemetry data to the Destination(s). + // The argument is non-const because the Manager may add or remove + // data from the entry. + virtual Error dispatch(TelemetryInfo *Entry) = 0; + + // Register a Destination. + virtual void addDestination(std::unique_ptr Destination) = 0; +}; + +} // namespace telemetry +} // namespace llvm + +#endif // LLVM_TELEMETRY_TELEMETRY_H diff --git a/llvm/lib/CMakeLists.txt b/llvm/lib/CMakeLists.txt index 503c77cb13bd0..f6465612d30c0 100644 --- a/llvm/lib/CMakeLists.txt +++ b/llvm/lib/CMakeLists.txt @@ -41,6 +41,7 @@ add_subdirectory(ProfileData) add_subdirectory(Passes) add_subdirectory(TargetParser) add_subdirectory(TextAPI) +add_subdirectory(Telemetry) add_subdirectory(ToolDrivers) add_subdirectory(XRay) if (LLVM_INCLUDE_TESTS) diff --git a/llvm/lib/Telemetry/CMakeLists.txt b/llvm/lib/Telemetry/CMakeLists.txt new file mode 100644 index 0000000000000..8208bdadb05e9 --- /dev/null +++ b/llvm/lib/Telemetry/CMakeLists.txt @@ -0,0 +1,6 @@ +add_llvm_component_library(LLVMTelemetry + Telemetry.cpp + + ADDITIONAL_HEADER_DIRS + "${LLVM_MAIN_INCLUDE_DIR}/llvm/Telemetry" +) diff --git a/llvm/lib/Telemetry/Telemetry.cpp b/llvm/lib/Telemetry/Telemetry.cpp new file mode 100644 index 0000000000000..e7640a64195bf --- /dev/null +++ b/llvm/lib/Telemetry/Telemetry.cpp @@ -0,0 +1,11 @@ +#include "llvm/Telemetry/Telemetry.h" + +namespace llvm { +namespace telemetry { + +void TelemetryInfo::serialize(Serializer &serializer) const { + serializer.write("SessionId", SessionId); +} + +} // namespace telemetry +} // namespace llvm diff --git a/llvm/unittests/CMakeLists.txt b/llvm/unittests/CMakeLists.txt index 911ede701982f..9d6b3999c4395 100644 --- a/llvm/unittests/CMakeLists.txt +++ b/llvm/unittests/CMakeLists.txt @@ -49,6 +49,7 @@ add_subdirectory(Support) add_subdirectory(TableGen) add_subdirectory(Target) add_subdirectory(TargetParser) +add_subdirectory(Telemetry) add_subdirectory(Testing) add_subdirectory(TextAPI) add_subdirectory(Transforms) diff --git a/llvm/unittests/Telemetry/CMakeLists.txt b/llvm/unittests/Telemetry/CMakeLists.txt new file mode 100644 index 0000000000000..a40ae4b2f5560 --- /dev/null +++ b/llvm/unittests/Telemetry/CMakeLists.txt @@ -0,0 +1,9 @@ +set(LLVM_LINK_COMPONENTS + Telemetry + Core + Support + ) + +add_llvm_unittest(TelemetryTests + TelemetryTest.cpp + ) diff --git a/llvm/unittests/Telemetry/TelemetryTest.cpp b/llvm/unittests/Telemetry/TelemetryTest.cpp new file mode 100644 index 0000000000000..173976bdb1650 --- /dev/null +++ b/llvm/unittests/Telemetry/TelemetryTest.cpp @@ -0,0 +1,678 @@ +//===- llvm/unittest/Telemetry/TelemetryTest.cpp - Telemetry unittests ---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Telemetry/Telemetry.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/JSON.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/raw_ostream.h" +#include "gtest/gtest.h" +#include +#include +#include + +// Testing parameters. +// These are set by each test to force certain outcomes. +// Since the tests may run in parallel, each test will have +// its own TestContext populated. +struct TestContext { + // Controlling whether there should be an Exit error (if so, what the + // expected exit message/description should be). + bool HasExitError = false; + std::string ExitMsg = ""; + + // Controllilng whether there is a vendor-provided config for + // Telemetry. + bool HasVendorConfig = false; + + // Controlling whether the data should be sanitized. + bool SanitizeData = false; + + // These two fields data emitted by the framework for later + // verifications by the tests. + std::string Buffer = ""; + std::vector EmittedJsons; + + // The expected Uuid generated by the fake tool. + std::string ExpectedUuid = ""; +}; + +// This is set by the test body. +static thread_local TestContext *CurrentContext = nullptr; + +namespace llvm { +namespace telemetry { +namespace vendor_code { + +// Generate unique (but deterministic "uuid" for testing purposes). +static std::string nextUuid() { + static std::atomic seed = 1111; + return std::to_string(seed.fetch_add(1, std::memory_order_acquire)); +} + +struct VendorEntryKind { + static const KindType VendorCommon = 168; // 0b010101000 + static const KindType Startup = 169; // 0b010101001 + static const KindType Exit = 170; // 0b010101010 +}; + +// Demonstrates that the TelemetryInfo (data courier) struct can be extended +// by downstream code to store additional data as needed. +// It can also define additional data serialization method. +struct VendorCommonTelemetryInfo : public TelemetryInfo { + static bool classof(const TelemetryInfo *T) { + if (T == nullptr) + return false; + // Subclasses of this is also acceptable. + return (T->getKind() & VendorEntryKind::VendorCommon) == + VendorEntryKind::VendorCommon; + } + + KindType getKind() const override { return VendorEntryKind::VendorCommon; } + + virtual void serializeToStream(llvm::raw_ostream &OS) const = 0; +}; + +struct StartupEvent : public VendorCommonTelemetryInfo { + std::string MagicStartupMsg; + + StartupEvent() = default; + StartupEvent(const StartupEvent &E) { + SessionId = E.SessionId; + Stats = E.Stats; + ExitDesc = E.ExitDesc; + Counter = E.Counter; + + MagicStartupMsg = E.MagicStartupMsg; + } + + static bool classof(const TelemetryInfo *T) { + if (T == nullptr) + return false; + return T->getKind() == VendorEntryKind::Startup; + } + + KindType getKind() const override { return VendorEntryKind::Startup; } + + void serializeToStream(llvm::raw_ostream &OS) const override { + OS << "SessionId:" << SessionId << "\n"; + OS << "MagicStartupMsg:" << MagicStartupMsg << "\n"; + } + + json::Object serializeToJson() const override { + return json::Object{ + {"Startup", + {{"SessionId", SessionId}, {"MagicStartupMsg", MagicStartupMsg}}}, + }; + } +}; + +struct ExitEvent : public VendorCommonTelemetryInfo { + std::string MagicExitMsg; + + ExitEvent() = default; + // Provide a copy ctor because we may need to make a copy + // before sanitizing the Entry. + ExitEvent(const ExitEvent &E) { + SessionId = E.SessionId; + Stats = E.Stats; + ExitDesc = E.ExitDesc; + Counter = E.Counter; + + MagicExitMsg = E.MagicExitMsg; + } + + static bool classof(const TelemetryInfo *T) { + if (T == nullptr) + return false; + return T->getKind() == VendorEntryKind::Exit; + } + + unsigned getKind() const override { return VendorEntryKind::Exit; } + + void serializeToStream(llvm::raw_ostream &OS) const override { + OS << "SessionId:" << SessionId << "\n"; + if (ExitDesc.has_value()) + OS << "ExitCode:" << ExitDesc->ExitCode << "\n"; + OS << "MagicExitMsg:" << MagicExitMsg << "\n"; + } + + json::Object serializeToJson() const override { + json::Array I = json::Array{ + {"SessionId", SessionId}, + {"MagicExitMsg", MagicExitMsg}, + }; + if (ExitDesc.has_value()) + I.push_back(json::Value({"ExitCode", ExitDesc->ExitCode})); + return json::Object{ + {"Exit", std::move(I)}, + }; + } +}; + +struct CustomTelemetryEvent : public VendorCommonTelemetryInfo { + std::vector Msgs; + + CustomTelemetryEvent() = default; + CustomTelemetryEvent(const CustomTelemetryEvent &E) { + SessionId = E.SessionId; + Stats = E.Stats; + ExitDesc = E.ExitDesc; + Counter = E.Counter; + + Msgs = E.Msgs; + } + + void serializeToStream(llvm::raw_ostream &OS) const override { + OS << "SessionId:" << SessionId << "\n"; + int I = 0; + for (const std::string &M : Msgs) { + OS << "MSG_" << I << ":" << M << "\n"; + ++I; + } + } + + json::Object serializeToJson() const override { + json::Object Inner; + Inner.try_emplace("SessionId", SessionId); + int I = 0; + for (const std::string &M : Msgs) { + Inner.try_emplace(("MSG_" + llvm::Twine(I)).str(), M); + ++I; + } + + return json::Object{{"Midpoint", std::move(Inner)}}; + } +}; + +// The following classes demonstrate how downstream code can +// define one or more custom Destination(s) to handle +// Telemetry data differently, specifically: +// + which data to send (fullset or sanitized) +// + where to send the data +// + in what form + +const std::string STRING_DEST("STRING"); +const std::string JSON_DEST("JSON"); + +// This Destination sends data to a std::string given at ctor. +class StringDestination : public Destination { +public: + // ShouldSanitize: if true, sanitize the data before emitting, otherwise, emit + // the full set. + StringDestination(bool ShouldSanitize, std::string &Buf) + : ShouldSanitize(ShouldSanitize), OS(Buf) {} + + Error emitEntry(const TelemetryInfo *Entry) override { + if (isa(Entry)) { + if (auto *E = dyn_cast(Entry)) { + if (ShouldSanitize) { + if (isa(E) || isa(E)) { + // There is nothing to sanitize for this type of data, so keep + // as-is. + E->serializeToStream(OS); + } else if (isa(E)) { + auto Sanitized = sanitizeFields(dyn_cast(E)); + Sanitized.serializeToStream(OS); + } else { + llvm_unreachable("unexpected type"); + } + } else { + E->serializeToStream(OS); + } + } + } else { + // Unfamiliar entries, just send the entry's UUID + OS << "SessionId:" << Entry->SessionId << "\n"; + } + return Error::success(); + } + + std::string name() const override { return STRING_DEST; } + +private: + // Returns a copy of the given entry, but with some fields sanitized. + CustomTelemetryEvent sanitizeFields(const CustomTelemetryEvent *Entry) { + CustomTelemetryEvent Sanitized(*Entry); + // Pretend that messages stored at ODD positions are "sensitive", + // hence need to be sanitized away. + int S = Sanitized.Msgs.size() - 1; + for (int I = S % 2 == 0 ? S - 1 : S; I >= 0; I -= 2) + Sanitized.Msgs[I] = ""; + return Sanitized; + } + + bool ShouldSanitize; + llvm::raw_string_ostream OS; +}; + +// This Destination sends data to some "blackbox" in form of JSON. +class JsonStreamDestination : public Destination { +public: + JsonStreamDestination(bool ShouldSanitize) : ShouldSanitize(ShouldSanitize) {} + + Error emitEntry(const TelemetryInfo *Entry) override { + if (auto *E = dyn_cast(Entry)) { + if (ShouldSanitize) { + if (isa(E) || isa(E)) { + // There is nothing to sanitize for this type of data, so keep as-is. + return SendToBlackbox(E->serializeToJson()); + } else if (isa(E)) { + auto Sanitized = sanitizeFields(dyn_cast(E)); + return SendToBlackbox(Sanitized.serializeToJson()); + } else { + llvm_unreachable("unexpected type"); + } + } else { + return SendToBlackbox(E->serializeToJson()); + } + } else { + // Unfamiliar entries, just send the entry's ID + return SendToBlackbox(json::Object{{"SessionId", Entry->SessionId}}); + } + return make_error("unhandled codepath in emitEntry", + inconvertibleErrorCode()); + } + + std::string name() const override { return JSON_DEST; } + +private: + // Returns a copy of the given entry, but with some fields sanitized. + CustomTelemetryEvent sanitizeFields(const CustomTelemetryEvent *Entry) { + CustomTelemetryEvent Sanitized(*Entry); + // Pretend that messages stored at EVEN positions are "sensitive", + // hence need to be sanitized away. + int S = Sanitized.Msgs.size() - 1; + for (int I = S % 2 == 0 ? S : S - 1; I >= 0; I -= 2) + Sanitized.Msgs[I] = ""; + + return Sanitized; + } + + llvm::Error SendToBlackbox(json::Object O) { + // Here is where the vendor-defined Destination class can + // send the data to some internal storage. + // For testing purposes, we just queue up the entries to + // the vector for validation. + CurrentContext->EmittedJsons.push_back(std::move(O)); + return Error::success(); + } + bool ShouldSanitize; +}; + +// Custom vendor-defined Telemeter that has additional data-collection point. +class TestTelemeter : public Telemeter { +public: + TestTelemeter(std::string SessionId) : Uuid(SessionId), Counter(0) {} + + static std::unique_ptr createInstance(Config *config) { + if (!config->EnableTelemetry) + return nullptr; + CurrentContext->ExpectedUuid = nextUuid(); + std::unique_ptr Telemeter = + std::make_unique(CurrentContext->ExpectedUuid); + // Set up Destination based on the given config. + for (const std::string &Dest : config->AdditionalDestinations) { + // The destination(s) are ALSO defined by vendor, so it should understand + // what the name of each destination signifies. + if (Dest == JSON_DEST) { + Telemeter->addDestination(new vendor_code::JsonStreamDestination( + CurrentContext->SanitizeData)); + } else if (Dest == STRING_DEST) { + Telemeter->addDestination(new vendor_code::StringDestination( + CurrentContext->SanitizeData, CurrentContext->Buffer)); + } else { + llvm_unreachable( + llvm::Twine("unknown destination: ", Dest).str().c_str()); + } + } + return Telemeter; + } + + void logStartup(llvm::StringRef ToolPath, TelemetryInfo *Entry) override { + ToolName = ToolPath.str(); + + // The vendor can add additional stuff to the entry before logging. + if (auto *S = dyn_cast(Entry)) { + S->MagicStartupMsg = llvm::Twine("Startup_", ToolPath).str(); + } + emitToDestinations(Entry); + } + + void logExit(llvm::StringRef ToolPath, TelemetryInfo *Entry) override { + // Ensure we're shutting down the same tool we started with. + if (ToolPath != ToolName) { + std::string Str; + raw_string_ostream OS(Str); + OS << "Expected tool with name" << ToolName << ", but got " << ToolPath; + llvm_unreachable(Str.c_str()); + } + + // The vendor can add additional stuff to the entry before logging. + if (auto *E = dyn_cast(Entry)) { + E->MagicExitMsg = llvm::Twine("Exit_", ToolPath).str(); + } + + emitToDestinations(Entry); + } + + void addDestination(Destination *Dest) override { + Destinations.push_back(Dest); + } + + void logMidpoint(TelemetryInfo *Entry) { + // The custom Telemeter can record and send additional data. + if (auto *C = dyn_cast(Entry)) { + C->Msgs.push_back("Two"); + C->Msgs.push_back("Deux"); + C->Msgs.push_back("Zwei"); + } + + emitToDestinations(Entry); + } + + const std::string &getUuid() const { return Uuid; } + + ~TestTelemeter() { + for (auto *Dest : Destinations) + delete Dest; + } + + template T makeDefaultTelemetryInfo() { + T Ret; + Ret.SessionId = Uuid; + Ret.Counter = Counter++; + return Ret; + } + +private: + void emitToDestinations(TelemetryInfo *Entry) { + for (Destination *Dest : Destinations) { + llvm::Error err = Dest->emitEntry(Entry); + if (err) { + // Log it and move on. + } + } + } + + const std::string Uuid; + size_t Counter; + std::string ToolName; + std::vector Destinations; +}; + +// Pretend to be a "weakly" defined vendor-specific function. +void ApplyVendorSpecificConfigs(Config *config) { + config->EnableTelemetry = true; +} + +} // namespace vendor_code +} // namespace telemetry +} // namespace llvm + +namespace { + +void ApplyCommonConfig(llvm::telemetry::Config *config) { + // Any shareable configs for the upstream tool can go here. + // ..... +} + +std::shared_ptr GetTelemetryConfig() { + // Telemetry is disabled by default. + // The vendor can enable in their config. + auto Config = std::make_shared(); + Config->EnableTelemetry = false; + + ApplyCommonConfig(Config.get()); + + // Apply vendor specific config, if present. + // In principle, this would be a build-time param, configured by the vendor. + // Eg: + // + // #ifdef HAS_VENDOR_TELEMETRY_CONFIG + // llvm::telemetry::vendor_code::ApplyVendorSpecificConfigs(config.get()); + // #endif + // + // But for unit testing, we use the testing params defined at the top. + if (CurrentContext->HasVendorConfig) { + llvm::telemetry::vendor_code::ApplyVendorSpecificConfigs(Config.get()); + } + return Config; +} + +using namespace llvm; +using namespace llvm::telemetry; + +// For deterministic tests, pre-defined certain important time-points +// rather than using now(). +// +// Preset StartTime to EPOCH. +auto StartTime = std::chrono::time_point{}; +// Pretend the time it takes for the tool's initialization is EPOCH + 5 +// milliseconds +auto InitCompleteTime = StartTime + std::chrono::milliseconds(5); +auto MidPointTime = StartTime + std::chrono::milliseconds(10); +auto MidPointCompleteTime = MidPointTime + std::chrono::milliseconds(5); +// Preset ExitTime to EPOCH + 20 milliseconds +auto ExitTime = StartTime + std::chrono::milliseconds(20); +// Pretend the time it takes to complete tearing down the tool is 10 +// milliseconds. +auto ExitCompleteTime = ExitTime + std::chrono::milliseconds(10); + +void AtToolStart(std::string ToolName, vendor_code::TestTelemeter *T) { + vendor_code::StartupEvent Entry = + T->makeDefaultTelemetryInfo(); + Entry.Stats = {StartTime, InitCompleteTime}; + T->logStartup(ToolName, &Entry); +} + +void AtToolExit(std::string ToolName, vendor_code::TestTelemeter *T) { + vendor_code::ExitEvent Entry = + T->makeDefaultTelemetryInfo(); + Entry.Stats = {ExitTime, ExitCompleteTime}; + + if (CurrentContext->HasExitError) { + Entry.ExitDesc = {1, CurrentContext->ExitMsg}; + } + T->logExit(ToolName, &Entry); +} + +void AtToolMidPoint(vendor_code::TestTelemeter *T) { + vendor_code::CustomTelemetryEvent Entry = + T->makeDefaultTelemetryInfo(); + Entry.Stats = {MidPointTime, MidPointCompleteTime}; + T->logMidpoint(&Entry); +} + +// Helper function to print the given object content to string. +static std::string ValueToString(const json::Value *V) { + std::string Ret; + llvm::raw_string_ostream P(Ret); + P << *V; + return Ret; +} + +// Without vendor's implementation, telemetry is not enabled by default. +TEST(TelemetryTest, TelemetryDefault) { + // Preset some test params. + TestContext Context; + Context.HasVendorConfig = false; + CurrentContext = &Context; + + std::shared_ptr Config = GetTelemetryConfig(); + auto Tool = vendor_code::TestTelemeter::createInstance(Config.get()); + + EXPECT_EQ(nullptr, Tool.get()); +} + +TEST(TelemetryTest, TelemetryEnabled) { + const std::string ToolName = "TelemetryTest"; + + // Preset some test params. + TestContext Context; + Context.HasVendorConfig = true; + Context.SanitizeData = false; + Context.Buffer.clear(); + Context.EmittedJsons.clear(); + CurrentContext = &Context; + + std::shared_ptr Config = GetTelemetryConfig(); + + // Add some destinations + Config->AdditionalDestinations.push_back(vendor_code::STRING_DEST); + Config->AdditionalDestinations.push_back(vendor_code::JSON_DEST); + + auto Tool = vendor_code::TestTelemeter::createInstance(Config.get()); + + AtToolStart(ToolName, Tool.get()); + AtToolMidPoint(Tool.get()); + AtToolExit(ToolName, Tool.get()); + + // Check that the Tool uses the expected UUID. + EXPECT_STREQ(Tool->getUuid().c_str(), CurrentContext->ExpectedUuid.c_str()); + + // Check that the StringDestination emitted properly + { + std::string ExpectedBuffer = + ("SessionId:" + llvm::Twine(CurrentContext->ExpectedUuid) + "\n" + + "MagicStartupMsg:Startup_" + llvm::Twine(ToolName) + "\n" + + "SessionId:" + llvm::Twine(CurrentContext->ExpectedUuid) + "\n" + + "MSG_0:Two\n" + "MSG_1:Deux\n" + "MSG_2:Zwei\n" + + "SessionId:" + llvm::Twine(CurrentContext->ExpectedUuid) + "\n" + + "MagicExitMsg:Exit_" + llvm::Twine(ToolName) + "\n") + .str(); + + EXPECT_STREQ(ExpectedBuffer.c_str(), CurrentContext->Buffer.c_str()); + } + + // Check that the JsonDestination emitted properly + { + + // There should be 3 events emitted by the Telemeter (start, midpoint, exit) + EXPECT_EQ(static_cast(3), CurrentContext->EmittedJsons.size()); + + const json::Value *StartupEntry = + CurrentContext->EmittedJsons[0].get("Startup"); + ASSERT_NE(StartupEntry, nullptr); + EXPECT_STREQ( + ("[[\"SessionId\",\"" + llvm::Twine(CurrentContext->ExpectedUuid) + + "\"],[\"MagicStartupMsg\",\"Startup_" + llvm::Twine(ToolName) + "\"]]") + .str() + .c_str(), + ValueToString(StartupEntry).c_str()); + + const json::Value *MidpointEntry = + CurrentContext->EmittedJsons[1].get("Midpoint"); + ASSERT_NE(MidpointEntry, nullptr); + // TODO: This is a bit flaky in that the json string printer sort the + // entries (for now), so the "UUID" field is put at the end of the array + // even though it was emitted first. + EXPECT_STREQ(("{\"MSG_0\":\"Two\",\"MSG_1\":\"Deux\",\"MSG_2\":\"Zwei\"," + "\"SessionId\":\"" + + llvm::Twine(CurrentContext->ExpectedUuid) + "\"}") + .str() + .c_str(), + ValueToString(MidpointEntry).c_str()); + + const json::Value *ExitEntry = CurrentContext->EmittedJsons[2].get("Exit"); + ASSERT_NE(ExitEntry, nullptr); + EXPECT_STREQ( + ("[[\"SessionId\",\"" + llvm::Twine(CurrentContext->ExpectedUuid) + + "\"],[\"MagicExitMsg\",\"Exit_" + llvm::Twine(ToolName) + "\"]]") + .str() + .c_str(), + ValueToString(ExitEntry).c_str()); + } +} + +// Similar to previous tests, but toggling the data-sanitization option ON. +// The recorded data should have some fields removed. +TEST(TelemetryTest, TelemetryEnabledSanitizeData) { + const std::string ToolName = "TelemetryTest_SanitizedData"; + + // Preset some test params. + TestContext Context; + Context.HasVendorConfig = true; + Context.SanitizeData = true; + Context.Buffer.clear(); + Context.EmittedJsons.clear(); + CurrentContext = &Context; + + std::shared_ptr Config = GetTelemetryConfig(); + + // Add some destinations + Config->AdditionalDestinations.push_back(vendor_code::STRING_DEST); + Config->AdditionalDestinations.push_back(vendor_code::JSON_DEST); + + auto Tool = vendor_code::TestTelemeter::createInstance(Config.get()); + + AtToolStart(ToolName, Tool.get()); + AtToolMidPoint(Tool.get()); + AtToolExit(ToolName, Tool.get()); + + // Check that the StringDestination emitted properly + { + // The StringDestination should have removed the odd-positioned msgs. + std::string ExpectedBuffer = + ("SessionId:" + llvm::Twine(CurrentContext->ExpectedUuid) + "\n" + + "MagicStartupMsg:Startup_" + llvm::Twine(ToolName) + "\n" + + "SessionId:" + llvm::Twine(CurrentContext->ExpectedUuid) + "\n" + + "MSG_0:Two\n" + "MSG_1:\n" + // <<< was sanitized away. + "MSG_2:Zwei\n" + + "SessionId:" + llvm::Twine(CurrentContext->ExpectedUuid) + "\n" + + "MagicExitMsg:Exit_" + llvm::Twine(ToolName) + "\n") + .str(); + EXPECT_STREQ(ExpectedBuffer.c_str(), CurrentContext->Buffer.c_str()); + } + + // Check that the JsonDestination emitted properly + { + + // There should be 3 events emitted by the Telemeter (start, midpoint, exit) + EXPECT_EQ(static_cast(3), CurrentContext->EmittedJsons.size()); + + const json::Value *StartupEntry = + CurrentContext->EmittedJsons[0].get("Startup"); + ASSERT_NE(StartupEntry, nullptr); + EXPECT_STREQ( + ("[[\"SessionId\",\"" + llvm::Twine(CurrentContext->ExpectedUuid) + + "\"],[\"MagicStartupMsg\",\"Startup_" + llvm::Twine(ToolName) + "\"]]") + .str() + .c_str(), + ValueToString(StartupEntry).c_str()); + + const json::Value *MidpointEntry = + CurrentContext->EmittedJsons[1].get("Midpoint"); + ASSERT_NE(MidpointEntry, nullptr); + // The JsonDestination should have removed the even-positioned msgs. + EXPECT_STREQ( + ("{\"MSG_0\":\"\",\"MSG_1\":\"Deux\",\"MSG_2\":\"\",\"SessionId\":\"" + + llvm::Twine(CurrentContext->ExpectedUuid) + "\"}") + .str() + .c_str(), + ValueToString(MidpointEntry).c_str()); + + const json::Value *ExitEntry = CurrentContext->EmittedJsons[2].get("Exit"); + ASSERT_NE(ExitEntry, nullptr); + EXPECT_STREQ( + ("[[\"SessionId\",\"" + llvm::Twine(CurrentContext->ExpectedUuid) + + "\"],[\"MagicExitMsg\",\"Exit_" + llvm::Twine(ToolName) + "\"]]") + .str() + .c_str(), + ValueToString(ExitEntry).c_str()); + } +} + +} // namespace