Add support for ondemand sourcefile fetch using python callback

Summary: 

Test Plan: 

Reviewers: 

Subscribers: 

Tasks: 

Tags: 


Differential Revision: https://phabricator.intern.facebook.com/D66931972

Author: Rahul Reddy Chamala

lldb/bindings/python/python-typemaps.swig

@@ -696,3 +696,47 @@ template <> bool SetNumberFromPyObject<double>(double &number, PyObject *obj) {
   $1 = $input == Py_None;
   $1 = $1 || PyCallable_Check(reinterpret_cast<PyObject *>($input));
 }
+
+// For lldb::SBPlatformResolveSourceFileCallback
+%typemap(in) (lldb::SBPlatformResolveSourceFileCallback callback, void *callback_baton) {
+  if (!($input == Py_None ||
+       PyCallable_Check(reinterpret_cast<PyObject *>($input)))) {
+        PyErr_SetString(PyExc_TypeError, "Need a callable object or None!");
+        SWIG_fail;
+       }
+
+  if ($input == Py_None) {
+    $1 = nullptr;
+    $2 = nullptr;
+  } else {
+    PythonCallable callable = Retain<PythonCallable>($input);
+    if (!callable.IsValid()) {
+      PyErr_SetString(PyExc_TypeError, "Need a valid callable object");
+      SWIG_fail;
+    }
+
+  llvm::Expected<PythonCallable::ArgInfo> arg_info = callable.GetArgInfo();
+  if (!arg_info) {
+    PyErr_SetString(PyExc_TypeError,
+                      ("Could not get arguments: " +
+                          llvm::toString(arg_info.takeError())).c_str());
+      SWIG_fail;
+  }
+
+  if (arg_info.get().max_positional_args != 3) {
+    PyErr_SetString(PyExc_TypeError, "Expected 3 argument callable object");
+    SWIG_fail;
+  }
+
+    Py_INCREF($input);
+
+    $1 = LLDBSwigPythonCallResolveSourceFileCallback;
+    $2 = $input;
+  }
+}
+
+%typemap(typecheck) (lldb::SBPlatformResolveSourceFileCallback callback,
+                     void *callback_baton) {
+  $1 = $input == Py_None;
+  $1 = $1 || PyCallable_Check(reinterpret_cast<PyObject *>($input));
+}

lldb/bindings/python/python-wrapper.swig

@@ -844,7 +844,7 @@ bool lldb_private::python::SWIGBridge::LLDBSwigPythonCallParsedCommandObject(
   auto pfunc = self.ResolveName<PythonCallable>("__call__");
 
   if (!pfunc.IsAllocated()) {
-    cmd_retobj.AppendError("Could not find '__call__' method in implementation class"); 
+    cmd_retobj.AppendError("Could not find '__call__' method in implementation class");
     return false;
   }
 
@@ -1148,4 +1148,50 @@ static SBError LLDBSwigPythonCallLocateModuleCallback(
 
   return *sb_error_ptr;
 }
+
+static SBError LLDBSwigPythonCallResolveSourceFileCallback(
+    void *callback_baton,
+    const char* build_id,
+    const SBFileSpec &original_source_file_spec_sb,
+    SBFileSpec &resolved_source_file_spec_sb) {
+  SWIG_Python_Thread_Block swig_thread_block;
+
+  PyErr_Cleaner py_err_cleaner(true);
+
+  PyObject *py_build_id = PyUnicode_FromString(build_id);
+  PythonObject build_id_arg(PyRefType::Borrowed,py_build_id);
+  PythonObject original_source_file_spec_arg = SWIGBridge::ToSWIGWrapper(
+      std::make_unique<SBFileSpec>(original_source_file_spec_sb));
+  PythonObject resolved_source_file_spec_arg = SWIGBridge::ToSWIGWrapper(
+      std::make_unique<SBFileSpec>(resolved_source_file_spec_sb));
+
+  PythonCallable callable =
+      Retain<PythonCallable>(reinterpret_cast<PyObject *>(callback_baton));
+  if (!callable.IsValid()) {
+    return SBError("The callback callable is not valid.");
+  }
+
+  PythonObject result = callable(build_id_arg, original_source_file_spec_arg,
+                                 resolved_source_file_spec_arg);
+
+  if (!result.IsAllocated())
+    return SBError("No result.");
+  lldb::SBError *sb_error_ptr = nullptr;
+  if (SWIG_ConvertPtr(result.get(), (void **)&sb_error_ptr,
+                      SWIGTYPE_p_lldb__SBError, 0) == -1) {
+    return SBError("Result is not SBError.");
+  }
+
+  if (sb_error_ptr->Success()) {
+    lldb::SBFileSpec *sb_resolved_source_file_spec_ptr = nullptr;
+    if (SWIG_ConvertPtr(resolved_source_file_spec_arg.get(),
+                        (void **)&sb_resolved_source_file_spec_ptr,
+                        SWIGTYPE_p_lldb__SBFileSpec, 0) == -1)
+      return SBError("resolved_source_file_spec is not SBFileSpec.");
+
+    resolved_source_file_spec_sb = *sb_resolved_source_file_spec_ptr;
+  }
+
+  return *sb_error_ptr;
+}
 %}

lldb/include/lldb/API/SBDefines.h

@@ -12,6 +12,7 @@
 #include "lldb/lldb-defines.h"
 #include "lldb/lldb-enumerations.h"
 #include "lldb/lldb-forward.h"
+#include "lldb/lldb-private.h"
 #include "lldb/lldb-types.h"
 #include "lldb/lldb-versioning.h"
 
@@ -141,9 +142,14 @@ typedef bool (*SBBreakpointHitCallback)(void *baton, SBProcess &process,
 typedef void (*SBDebuggerDestroyCallback)(lldb::user_id_t debugger_id,
                                           void *baton);
 
-typedef SBError (*SBPlatformLocateModuleCallback)(
+typedef lldb::SBError (*SBPlatformLocateModuleCallback)(
     void *baton, const SBModuleSpec &module_spec, SBFileSpec &module_file_spec,
     SBFileSpec &symbol_file_spec);
+
+typedef lldb::SBError (*SBPlatformResolveSourceFileCallback)(
+    void *baton, const char *buildId,
+    const SBFileSpec &original_source_file_spec,
+    SBFileSpec &resolved_source_file_spec);
 }
 
 #endif // LLDB_API_SBDEFINES_H

lldb/include/lldb/API/SBPlatform.h

@@ -190,6 +190,9 @@ class LLDB_API SBPlatform {
   SBError SetLocateModuleCallback(lldb::SBPlatformLocateModuleCallback callback,
                                   void *callback_baton);
 
+  SBError SetResolveSourceFileCallback(
+      lldb::SBPlatformResolveSourceFileCallback callback, void *callback_baton);
+
 protected:
   friend class SBDebugger;
   friend class SBTarget;

lldb/include/lldb/Symbol/LineEntry.h

@@ -128,7 +128,7 @@ struct LineEntry {
   ///
   /// \param[in] target_sp
   ///     Shared pointer to the target this LineEntry belongs to.
-  void ApplyFileMappings(lldb::TargetSP target_sp);
+  void ApplyFileMappings(lldb::TargetSP target_sp, const Address &address);
 
   /// Helper to access the file.
   const FileSpec &GetFile() const { return file_sp->GetSpecOnly(); }
@@ -181,6 +181,7 @@ struct LineEntry {
 ///     Returns \b true if lhs < rhs, false otherwise.
 bool operator<(const LineEntry &lhs, const LineEntry &rhs);
 
+// Add signature for CallFetchSourceFileCallBack
 } // namespace lldb_private
 
 #endif // LLDB_SYMBOL_LINEENTRY_H

lldb/include/lldb/Target/Platform.h

@@ -329,6 +329,10 @@ class Platform : public PluginInterface {
   virtual bool GetModuleSpec(const FileSpec &module_file_spec,
                              const ArchSpec &arch, ModuleSpec &module_spec);
 
+  void CallResolveSourceFileCallbackIfSet(
+      const char *build_id, const FileSpec &original_source_file_spec,
+      FileSpec &resolved_source_file_spec, bool *did_create_ptr);
+
   virtual Status ConnectRemote(Args &args);
 
   virtual Status DisconnectRemote();
@@ -921,6 +925,11 @@ class Platform : public PluginInterface {
                                FileSpec &symbol_file_spec)>
       LocateModuleCallback;
 
+  typedef std::function<Status(const char *buildId,
+                               const FileSpec &original_fileSpec,
+                               FileSpec &newFileSpec)>
+      ResolveSourceFileCallback;
+
   /// Set locate module callback. This allows users to implement their own
   /// module cache system. For example, to leverage artifacts of build system,
   /// to bypass pulling files from remote platform, or to search symbol files
@@ -929,6 +938,10 @@ class Platform : public PluginInterface {
 
   LocateModuleCallback GetLocateModuleCallback() const;
 
+  void SetResolveSourceFileCallback(ResolveSourceFileCallback callback);
+
+  ResolveSourceFileCallback GetResolveSourceFileCallback() const;
+
 protected:
   /// Create a list of ArchSpecs with the given OS and a architectures. The
   /// vendor field is left as an "unspecified unknown".
@@ -977,6 +990,7 @@ class Platform : public PluginInterface {
   bool m_calculated_trap_handlers;
   const std::unique_ptr<ModuleCache> m_module_cache;
   LocateModuleCallback m_locate_module_callback;
+  ResolveSourceFileCallback m_resolve_source_file_callback;
 
   /// Ask the Platform subclass to fill in the list of trap handler names
   ///

lldb/source/API/SBPlatform.cpp

@@ -734,3 +734,39 @@ SBError SBPlatform::SetLocateModuleCallback(
       });
   return SBError();
 }
+
+SBError SBPlatform::SetResolveSourceFileCallback(
+    lldb::SBPlatformResolveSourceFileCallback callback, void *callback_baton) {
+  LLDB_INSTRUMENT_VA(this, callback, callback_baton);
+  PlatformSP platform_sp(GetSP());
+  if (!platform_sp) {
+    return SBError("invalid platform");
+  }
+
+  if (!callback) {
+    // Clear the callback.
+    platform_sp->SetResolveSourceFileCallback(nullptr);
+    return SBError("invalid callback");
+  }
+
+  // Platform.h does not accept lldb::SBPlatformFetchSourceFileCallback
+  // directly because of the SBFileSpec dependencies. Use a lambda to convert
+  // FileSpec <--> SBFileSpec for the callback arguments.
+  platform_sp->SetResolveSourceFileCallback(
+      [callback, callback_baton](const char *build_id,
+                                 const FileSpec &original_source_file_spec,
+                                 FileSpec &resolved_source_file_spec) {
+        SBFileSpec resolved_source_file_spec_sb(resolved_source_file_spec);
+
+        SBError error =
+            callback(callback_baton, build_id, original_source_file_spec,
+                     resolved_source_file_spec_sb);
+
+        if (error.Success()) {
+          resolved_source_file_spec = resolved_source_file_spec_sb.ref();
+        }
+
+        return error.ref();
+      });
+  return SBError();
+}

lldb/source/Symbol/LineEntry.cpp

@@ -7,7 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "lldb/Symbol/LineEntry.h"
+#include "lldb/Core/Module.h"
 #include "lldb/Symbol/CompileUnit.h"
+#include "lldb/Symbol/SymbolContext.h"
+#include "lldb/Target/Platform.h"
 #include "lldb/Target/Process.h"
 #include "lldb/Target/Target.h"
 
@@ -240,8 +243,47 @@ AddressRange LineEntry::GetSameLineContiguousAddressRange(
   return complete_line_range;
 }
 
-void LineEntry::ApplyFileMappings(lldb::TargetSP target_sp) {
+void LineEntry::ApplyFileMappings(lldb::TargetSP target_sp,
+                                  const Address &address) {
   if (target_sp) {
+
+    SymbolContext sc;
+    target_sp->GetImages().ResolveSymbolContextForAddress(
+        address, lldb::eSymbolContextModule, sc);
+    lldb::ModuleSP module_sp = sc.module_sp;
+
+    auto spec = original_file_sp->GetSpecOnly();
+    std::string path = spec.GetPath();
+
+    if (module_sp) {
+      ObjectFile *obj_file = module_sp->GetObjectFile();
+      module_sp->GetFileSpec();
+      if (obj_file) {
+
+        UUID build_id = obj_file->GetUUID();
+        auto buildId_string = new std::string(build_id.GetAsString());
+
+        const char *build_id_ptr = buildId_string->c_str();
+
+        FileSpec newSpec;
+        // Fetches the new spec file
+        bool *didFetchSourceFile = new bool(false);
+
+        lldb::PlatformSP platform_sp = target_sp->GetPlatform();
+        if (!platform_sp)
+          return;
+
+        FileSpec resolved_source_file_spec;
+        platform_sp->CallResolveSourceFileCallbackIfSet(
+            build_id_ptr, spec, resolved_source_file_spec, didFetchSourceFile);
+        if (didFetchSourceFile) {
+          original_file_sp =
+              std::make_shared<SupportFile>(resolved_source_file_spec);
+          file_sp = std::make_shared<SupportFile>(resolved_source_file_spec);
+        }
+      }
+    }
+
     // Apply any file remappings to our file.
     if (auto new_file_spec = target_sp->GetSourcePathMap().FindFile(
             original_file_sp->GetSpecOnly()))

lldb/source/Target/Platform.cpp

@@ -1756,6 +1756,50 @@ void Platform::CallLocateModuleCallbackIfSet(const ModuleSpec &module_spec,
   }
 }
 
+void Platform::CallResolveSourceFileCallbackIfSet(
+    const char *build_id, const FileSpec &original_source_file_spec,
+    FileSpec &resolved_source_file_spec, bool *did_create_ptr) {
+  if (!m_resolve_source_file_callback) {
+    // Fetch source file callback is not set.
+    return;
+  }
+
+  FileSpec module_file_spec;
+  Status error = m_resolve_source_file_callback(
+      build_id, original_source_file_spec, resolved_source_file_spec);
+
+  // Fetch source file callback is set and called. Check the error.
+  Log *log = GetLog(LLDBLog::Platform);
+  if (error.Fail()) {
+    LLDB_LOGF(log, "%s: Fetch source file callback failed: %s",
+              LLVM_PRETTY_FUNCTION, error.AsCString());
+    return;
+  }
+
+  if (!resolved_source_file_spec) {
+    LLDB_LOGF(log,
+              "%s: fetch source file callback did not set "
+              "resolved_source_file_spec",
+              LLVM_PRETTY_FUNCTION);
+    return;
+  }
+
+  // If the callback returned a source file, it should exist.
+  if (resolved_source_file_spec &&
+      !FileSystem::Instance().Exists(resolved_source_file_spec)) {
+    LLDB_LOGF(log,
+              "%s: fetch source file callback set a non-existent file to "
+              "source_file_spec: %s",
+              LLVM_PRETTY_FUNCTION,
+              resolved_source_file_spec.GetPath().c_str());
+    // Clear source_file_spec for the error.
+    resolved_source_file_spec.Clear();
+    return;
+  }
+
+  *did_create_ptr = true;
+}
+
 bool Platform::GetCachedSharedModule(const ModuleSpec &module_spec,
                                      lldb::ModuleSP &module_sp,
                                      bool *did_create_ptr) {
@@ -2161,6 +2205,16 @@ Platform::LocateModuleCallback Platform::GetLocateModuleCallback() const {
   return m_locate_module_callback;
 }
 
+void Platform::SetResolveSourceFileCallback(
+    ResolveSourceFileCallback callback) {
+  m_resolve_source_file_callback = callback;
+}
+
+Platform::ResolveSourceFileCallback
+Platform::GetResolveSourceFileCallback() const {
+  return m_resolve_source_file_callback;
+}
+
 PlatformSP PlatformList::GetOrCreate(llvm::StringRef name) {
   std::lock_guard<std::recursive_mutex> guard(m_mutex);
   for (const PlatformSP &platform_sp : m_platforms) {

lldb/source/Target/StackFrame.cpp

@@ -395,7 +395,7 @@ StackFrame::GetSymbolContext(SymbolContextItem resolve_scope) {
         if ((resolved & eSymbolContextLineEntry) &&
             !m_sc.line_entry.IsValid()) {
           m_sc.line_entry = sc.line_entry;
-          m_sc.line_entry.ApplyFileMappings(m_sc.target_sp);
+          m_sc.line_entry.ApplyFileMappings(m_sc.target_sp, lookup_addr);
         }
       }
     } else {