[Draft] Resolution of symbols by the server

.gitignore

@@ -34,3 +34,6 @@ compile_commands.json
 profiler/build/wasm/Tracy-release.*
 profiler/build/wasm/Tracy-debug.*
 profiler/build/wasm/embed.tracy
+examples/ToyPathTracer/Windows/TestCpu
+examples/ToyPathTracer/Windows/x64
+examples/ToyPathTracer/Windows/TestCpu.vcxproj.user

CMakeLists.txt

@@ -177,17 +177,12 @@ set(client_includes
     ${TRACY_PUBLIC_DIR}/client/tracy_SPSCQueue.h
     ${TRACY_PUBLIC_DIR}/client/TracyKCore.hpp
     ${TRACY_PUBLIC_DIR}/client/TracyArmCpuTable.hpp
-    ${TRACY_PUBLIC_DIR}/client/TracyCallstack.h
-    ${TRACY_PUBLIC_DIR}/client/TracyCallstack.hpp
     ${TRACY_PUBLIC_DIR}/client/TracyCpuid.hpp
-    ${TRACY_PUBLIC_DIR}/client/TracyDebug.hpp
     ${TRACY_PUBLIC_DIR}/client/TracyDxt1.hpp
-    ${TRACY_PUBLIC_DIR}/client/TracyFastVector.hpp
     ${TRACY_PUBLIC_DIR}/client/TracyLock.hpp
     ${TRACY_PUBLIC_DIR}/client/TracyProfiler.hpp
     ${TRACY_PUBLIC_DIR}/client/TracyRingBuffer.hpp
     ${TRACY_PUBLIC_DIR}/client/TracyScoped.hpp
-    ${TRACY_PUBLIC_DIR}/client/TracyStringHelpers.hpp
     ${TRACY_PUBLIC_DIR}/client/TracySysPower.hpp
     ${TRACY_PUBLIC_DIR}/client/TracySysTime.hpp
     ${TRACY_PUBLIC_DIR}/client/TracySysTrace.hpp
@@ -199,13 +194,18 @@ set(common_includes
     ${TRACY_PUBLIC_DIR}/common/TracyAlign.hpp
     ${TRACY_PUBLIC_DIR}/common/TracyAlloc.hpp
     ${TRACY_PUBLIC_DIR}/common/TracyApi.h
+    ${TRACY_PUBLIC_DIR}/common/TracyCallstack.h
+    ${TRACY_PUBLIC_DIR}/common/TracyCallstack.hpp
     ${TRACY_PUBLIC_DIR}/common/TracyColor.hpp
+    ${TRACY_PUBLIC_DIR}/common/TracyDebug.hpp
+    ${TRACY_PUBLIC_DIR}/common/TracyFastVector.hpp
     ${TRACY_PUBLIC_DIR}/common/TracyForceInline.hpp
     ${TRACY_PUBLIC_DIR}/common/TracyMutex.hpp
     ${TRACY_PUBLIC_DIR}/common/TracyProtocol.hpp
     ${TRACY_PUBLIC_DIR}/common/TracyQueue.hpp
     ${TRACY_PUBLIC_DIR}/common/TracySocket.hpp
     ${TRACY_PUBLIC_DIR}/common/TracyStackFrames.hpp
+    ${TRACY_PUBLIC_DIR}/common/TracyStringHelpers.hpp
     ${TRACY_PUBLIC_DIR}/common/TracySystem.hpp
     ${TRACY_PUBLIC_DIR}/common/TracyUwp.hpp
     ${TRACY_PUBLIC_DIR}/common/TracyYield.hpp)

capture/src/capture.cpp

@@ -164,7 +164,10 @@ int main( int argc, char** argv )
 
     printf( "Connecting to %s:%i...", address, port );
     fflush( stdout );
-    tracy::Worker worker( address, port, memoryLimit );
+
+    tracy::Worker::SymbolResolutionConfig symConfig{};
+    symConfig.m_attemptResolutionByWorker = false;
+    tracy::Worker worker( address, port, memoryLimit, symConfig );
     while( !worker.HasData() )
     {
         const auto handshake = worker.GetHandshakeStatus();

cmake/server.cmake

@@ -3,6 +3,7 @@ set(TRACY_COMMON_DIR ${CMAKE_CURRENT_LIST_DIR}/../public/common)
 set(TRACY_COMMON_SOURCES
     tracy_lz4.cpp
     tracy_lz4hc.cpp
+    TracyCallstack.cpp
     TracySocket.cpp
     TracyStackFrames.cpp
     TracySystem.cpp

csvexport/src/csvexport.cpp

@@ -214,7 +214,9 @@ int main(int argc, char** argv)
         return 1;
     }
 
-    auto worker = tracy::Worker(*f);
+    tracy::Worker::SymbolResolutionConfig symConfig{};
+    symConfig.m_attemptResolutionByWorker = false;
+    auto worker = tracy::Worker(*f, symConfig);
 
     if (args.unwrapMessages) 
     {

import/src/import-chrome.cpp

@@ -346,7 +346,9 @@ int main( int argc, char** argv )
         return out;
     };
 
-    tracy::Worker worker( getFilename(output), getFilename(input), timeline, messages, plots, threadNames );
+    tracy::Worker::SymbolResolutionConfig symConfig{};
+    symConfig.m_attemptResolutionByWorker = false;  
+    tracy::Worker worker( getFilename(output), getFilename(input), timeline, messages, plots, threadNames, symConfig );
 
     auto w = std::unique_ptr<tracy::FileWrite>( tracy::FileWrite::Open( output, clev ) );
     if( !w )

import/src/import-fuchsia.cpp

@@ -665,9 +665,11 @@ int main(int argc, char **argv) {
       out--;
     return out;
   };
-
+
+  tracy::Worker::SymbolResolutionConfig symConfig{};
+  symConfig.m_attemptResolutionByWorker = false;
   tracy::Worker worker(getFilename(output), getFilename(input), timeline,
-                       messages, plots, std::move(dec.threadNames));
+                       messages, plots, std::move(dec.threadNames), symConfig);
 
   auto w =
       std::unique_ptr<tracy::FileWrite>(tracy::FileWrite::Open(output, clev));

manual/tracy.tex

@@ -1895,10 +1895,32 @@ \subsubsection{Debugging symbols}
 
 At initilization time, tracy will attempt to preload symbols for device drivers and process modules. As this process can be slow when a lot of pdbs are involved, you can set the \texttt{TRACY\_NO\_DBGHELP\_INIT\_LOAD} environment variable to "1" to disable this behavior and rely on-demand symbol loading.
 
+\begin{bclogo}[
+noborder=true,
+couleur=black!5,
+logo=\bcbombe
+]{Important}
+
+To be able to fully resolve kernel symbols, you might need to have matching versions of \texttt{dbghelp.dll} and \texttt{symsrv.dll} in your \texttt{PATH} or working directory and use symbol servers. Set the \texttt{\_NT\_SYMBOL\_PATH} environment variable to point to the symbol servers you want. For example:
+\begin{lstlisting}
+_NT\_SYMBOL\_PATH=srv*YourPdbCachePath*https://msdl.microsoft.com/download/symbols
+\end{lstlisting}
+
+To get more information about this topic, check Microsoft's official docs: \url{https://learn.microsoft.com/en-us/windows-hardware/drivers/debugger/symbol-path}.
+Note that downloading the kernel drivers symbols can be slow at the first execution of the program, or a Window update.
+\end{bclogo}
+
 \paragraph{Disabling resolution of inline frames}
 
 Inline frames retrieval on Windows can be multiple orders of magnitude slower than just performing essential symbol resolution. This manifests as profiler seemingly being stuck for a long time, having hundreds of thousands of query backlog entries queued, which are slowly trickling down. If your use case requires speed of operation rather than having call stacks with inline frames included, you may define the \texttt{TRACY\_NO\_CALLSTACK\_INLINES} macro, which will make the profiler stick to the basic but fast frame resolution mode.
 
+\paragraph{Server symbol resolution}
+
+Instead of having the client do the resolution, you can configure the server to attempt the symbol resolution.
+This is controlled by the profiler global settings in the "Symbol resolution" section.
+If "Attempt resolution by profiler" is enabled, the server will attempt to resolve symbols locally. For this, the Tracy-Client will send informations about the executable images loaded by the process. If symbol resolution on the server fails, it will query the client for the information as a fallback mechanism.
+However if "Prevent resolution by application" is enabled, Tracy-Client will not resolve any symbols and simply send the minimal images information.
+
 \paragraph{Offline symbol resolution}
 
 By default, tracy client resolves callstack symbols in a background thread at runtime.

meson.build

@@ -145,17 +145,12 @@ client_includes = [
     'public/client/tracy_rpmalloc.hpp',
     'public/client/tracy_SPSCQueue.h',
     'public/client/TracyArmCpuTable.hpp',
-    'public/client/TracyCallstack.h',
-    'public/client/TracyCallstack.hpp',
-    'public/client/TracyDebug.hpp',
     'public/client/TracyDxt1.hpp',
-    'public/client/TracyFastVector.hpp',
     'public/client/TracyKCore.hpp',
     'public/client/TracyLock.hpp',
     'public/client/TracyProfiler.hpp',
     'public/client/TracyRingBuffer.hpp',
     'public/client/TracyScoped.hpp',
-    'public/client/TracyStringHelpers.hpp',
     'public/client/TracySysPower.hpp',
     'public/client/TracySysTime.hpp',
     'public/client/TracySysTrace.hpp',
@@ -168,13 +163,18 @@ common_includes = [
     'public/common/TracyAlign.hpp',
     'public/common/TracyAlloc.hpp',
     'public/common/TracyApi.h',
+    'public/common/TracyCallstack.h',
+    'public/common/TracyCallstack.hpp',
     'public/common/TracyColor.hpp',
+    'public/common/TracyDebug.hpp',
+    'public/common/TracyFastVector.hpp',
     'public/common/TracyForceInline.hpp',
     'public/common/TracyMutex.hpp',
     'public/common/TracyProtocol.hpp',
     'public/common/TracyQueue.hpp',
     'public/common/TracySocket.hpp',
     'public/common/TracyStackFrames.hpp',
+    'public/common/TracyStringHelpers.hpp',
     'public/common/TracySystem.hpp',
     'public/common/TracyUwp.hpp',
     'public/common/TracyYield.hpp'

profiler/CMakeLists.txt

@@ -19,7 +19,9 @@ project(
 
 if(SELF_PROFILE)
     add_definitions(-DTRACY_ENABLE)
-    add_compile_options(-g -O3 -fno-omit-frame-pointer)
+    if(NOT MSVC)
+        add_compile_options(-g -O3 -fno-omit-frame-pointer)
+    endif()
 endif()
 
 include(${CMAKE_CURRENT_LIST_DIR}/../cmake/config.cmake)

profiler/src/main.cpp

@@ -237,6 +237,8 @@ static void LoadConfig()
     if( ini_sget( ini, "memory", "percent", "%d", &v ) && v >= 1 && v < 1000 ) s_config.memoryLimitPercent = v;
     if( ini_sget( ini, "achievements", "enabled", "%d", &v ) ) s_config.achievements = v;
     if( ini_sget( ini, "achievements", "asked", "%d", &v ) ) s_config.achievementsAsked = v;
+    if( ini_sget( ini, "symbols", "attemptResolutionByServer", "%d", &v ) ) s_config.symbolsAttemptResolutionByServer = v;
+    if( ini_sget( ini, "symbols", "preventResolutionByClient", "%d", &v ) ) s_config.symbolsPreventResolutionByClient = v;
 
     ini_free( ini );
 }
@@ -267,6 +269,10 @@ static bool SaveConfig()
     fprintf( f, "enabled = %i\n", (int)s_config.achievements );
     fprintf( f, "asked = %i\n", (int)s_config.achievementsAsked );
 
+    fprintf(f, "\n[symbols]\n");
+    fprintf(f, "attemptResolutionByServer = %i\n", (int)s_config.symbolsAttemptResolutionByServer);
+    fprintf(f, "preventResolutionByClient = %i\n", (int)s_config.symbolsPreventResolutionByClient);
+
     fclose( f );
     return true;
 }
@@ -840,6 +846,18 @@ static void DrawContents()
                 ImGui::Spacing();
                 if( ImGui::Checkbox( "Enable achievements", &s_config.achievements ) ) SaveConfig();
 
+                ImGui::TextUnformatted( "Symbol resolution" );
+                ImGui::Indent();
+                if( ImGui::Checkbox( "Attempt resolution by profiler", &s_config.symbolsAttemptResolutionByServer ) ) SaveConfig();
+                ImGui::SameLine();
+                tracy::DrawHelpMarker( "When enabled, the profiler will attempt to resolve symbols first before querying the application." );
+
+                if( ImGui::Checkbox( "Prevent resolution by application", &s_config.symbolsPreventResolutionByClient ) ) SaveConfig();
+                ImGui::SameLine();
+                tracy::DrawHelpMarker( "When enabled, the application will not attempt to resolve symbols at all. The profiler can do the resolution, or it will require to use the `update` tool." );
+
+                ImGui::Unindent();
+
                 ImGui::PopStyleVar();
                 ImGui::TreePop();
             }

profiler/src/profiler/TracyConfig.hpp

@@ -20,6 +20,8 @@ struct Config
     int dynamicColors = 1;
     bool forceColors = false;
     int shortenName = (int)ShortenName::NoSpaceAndNormalize;
+    bool symbolsAttemptResolutionByServer = true;
+    bool symbolsPreventResolutionByClient = false;
 };
 
 }

profiler/src/profiler/TracySourceView.cpp

@@ -1157,7 +1157,8 @@ void SourceView::RenderSymbolView( Worker& worker, View& view )
 
     const auto shortenName = view.GetShortenName();
     auto sym = worker.GetSymbolData( m_symAddr );
-    assert( sym );
+    if( sym == nullptr ) return; // Might no have received symbol info yet
+
     ImGui::PushFont( m_bigFont );
     ImGui::PushStyleVar( ImGuiStyleVar_FramePadding, ImVec2( 0, 0 ) );
     if( ButtonDisablable( " " ICON_FA_CARET_LEFT " ", m_historyCursor <= 1 ) )

profiler/src/profiler/TracyView.cpp

@@ -36,7 +36,7 @@ namespace tracy
 double s_time = 0;
 
 View::View( void(*cbMainThread)(const std::function<void()>&, bool), const char* addr, uint16_t port, ImFont* fixedWidth, ImFont* smallFont, ImFont* bigFont, SetTitleCallback stcb, SetScaleCallback sscb, AttentionCallback acb, const Config& config, AchievementsMgr* amgr )
-    : m_worker( addr, port, config.memoryLimit == 0 ? -1 : ( config.memoryLimitPercent * tracy::GetPhysicalMemorySize() / 100 ) )
+    : m_worker( addr, port, config.memoryLimit == 0 ? -1 : ( config.memoryLimitPercent * tracy::GetPhysicalMemorySize() / 100 ), Worker::SymbolResolutionConfig{ config.symbolsAttemptResolutionByServer, config.symbolsPreventResolutionByClient } )
     , m_staticView( false )
     , m_viewMode( ViewMode::LastFrames )
     , m_viewModeHeuristicTry( true )
@@ -66,10 +66,11 @@ View::View( void(*cbMainThread)(const std::function<void()>&, bool), const char*
 {
     InitTextEditor();
     SetupConfig( config );
+
 }
 
 View::View( void(*cbMainThread)(const std::function<void()>&, bool), FileRead& f, ImFont* fixedWidth, ImFont* smallFont, ImFont* bigFont, SetTitleCallback stcb, SetScaleCallback sscb, AttentionCallback acb, const Config& config, AchievementsMgr* amgr )
-    : m_worker( f )
+    : m_worker( f, { config.symbolsAttemptResolutionByServer, config.symbolsPreventResolutionByClient }, EventType::All, true, false )
     , m_filename( f.GetFilename() )
     , m_staticView( true )
     , m_viewMode( ViewMode::Paused )
@@ -912,6 +913,18 @@ bool View::DrawImpl()
     ImGui::SameLine();
     ToggleButton( ICON_FA_FINGERPRINT " Info", m_showInfo );
     ImGui::SameLine();
+
+    if( m_filename.empty() ) // Can't modify when loading from file
+    {
+        ImGui::SameLine();
+        if (ImGui::Button( ICON_FA_USER_SECRET " Resolve Symbols"))
+        {
+            // TODO: Make it async ?
+            m_worker.ResolveSymbolLocally();        
+        }
+
+    }
+
     if( ImGui::Button( ICON_FA_SCREWDRIVER_WRENCH ) ) ImGui::OpenPopup( "ToolsPopup" );
     if( ImGui::BeginPopup( "ToolsPopup" ) )
     {

profiler/src/profiler/TracyView_Compare.cpp

@@ -255,7 +255,9 @@ void View::DrawCompare()
                         m_compare.loadThread = std::thread( [this, f] {
                             try
                             {
-                                m_compare.second = std::make_unique<Worker>( *f, EventType::SourceCache );
+                                tracy::Worker::SymbolResolutionConfig symConfig{};
+                                symConfig.m_attemptResolutionByWorker = false;
+                                m_compare.second = std::make_unique<Worker>( *f, symConfig, EventType::SourceCache );
                                 m_compare.userData = std::make_unique<UserData>( m_compare.second->GetCaptureProgram().c_str(), m_compare.second->GetCaptureTime() );
                                 m_compare.diffDirection = m_worker.GetCaptureTime() < m_compare.second->GetCaptureTime();
                             }

public/TracyClient.cpp

@@ -21,7 +21,7 @@
 
 #include "common/tracy_lz4.cpp"
 #include "client/TracyProfiler.cpp"
-#include "client/TracyCallstack.cpp"
+#include "common/TracyCallstack.cpp"
 #include "client/TracySysPower.cpp"
 #include "client/TracySysTime.cpp"
 #include "client/TracySysTrace.cpp"
@@ -32,27 +32,8 @@
 #include "client/TracyOverride.cpp"
 #include "client/TracyKCore.cpp"
 
-#if defined(TRACY_HAS_CALLSTACK)
-#  if TRACY_HAS_CALLSTACK == 2 || TRACY_HAS_CALLSTACK == 3 || TRACY_HAS_CALLSTACK == 4 || TRACY_HAS_CALLSTACK == 6
-#    include "libbacktrace/alloc.cpp"
-#    include "libbacktrace/dwarf.cpp"
-#    include "libbacktrace/fileline.cpp"
-#    include "libbacktrace/mmapio.cpp"
-#    include "libbacktrace/posix.cpp"
-#    include "libbacktrace/sort.cpp"
-#    include "libbacktrace/state.cpp"
-#    if TRACY_HAS_CALLSTACK == 4
-#      include "libbacktrace/macho.cpp"
-#    else
-#      include "libbacktrace/elf.cpp"
-#    endif
-#    include "common/TracyStackFrames.cpp"
-#  endif
-#endif
-
 #ifdef _MSC_VER
 #  pragma comment(lib, "ws2_32.lib")
-#  pragma comment(lib, "dbghelp.lib")
 #  pragma comment(lib, "advapi32.lib")
 #  pragma comment(lib, "user32.lib")
 #  pragma warning(pop)