Enable Portable ThreadPool events to be written using NativeRuntimeEventSource (#47829)

* match up the GUID

* why is this returning nothing

* Make ThreadPool emit events via NativeRuntimeEventSource

* found error...

* hack around having duplicate events generated

* make portable threadpool fire events through nativeruntimeeventsource

* make the initializer live in correct namespace

* EventListener tests are passing now

* make native QCall endpoints that sink into LTTng

* send events through XplatEventLogger sink

* Fix merge error

* double registration

* fix callback not getting handled when lttng is enabled

* cleanup 1

* rename file

* remove stale comments

* remove stale comment

* undo useless change

* fix broken EventListener

* apply code review feedback

* fix windows build

* fix incorrect qcall

* fix up ifdefs for Windows builds

* fix ifdef

* put back removed code

* tests pass now

* Fix NativeRuntimeEventSource not getting enabled for lttng

* Add some more comments

* Add comment in generated code

* fix mono build

* PR feedback

* fix typo

* code review feedback

Author: sywhang

src/coreclr/scripts/genRuntimeEventSources.py

@@ -15,7 +15,7 @@
 /**********************************************************************
 
 DO NOT MODIFY. AUTOGENERATED FILE.
-This file is generated by <root>/src/scripts/genRuntimeEventSources.py
+This file is generated by <root>/src/coreclr/scripts/genRuntimeEventSources.py
 
 **********************************************************************/
 """
@@ -73,22 +73,30 @@ def getCSharpTypeFromManifestType(manifestType):
 
 def generateEvent(eventNode, providerNode, outputFile, stringTable):
 
+    # ThreadPool events are defined manually in NativeRuntimeEventSource.PortableThreadPool.cs
+    symbol = eventNode.getAttribute("symbol")
+    if "ThreadPool" in symbol:
+        return
+    
+    evtLevel = eventNode.getAttribute("level")[4:]
+    evtKeywords = ""
     # Write the event attribute.
-    writeOutput(outputFile, "[Event("+ eventNode.getAttribute("value") + ", Version = " + eventNode.getAttribute("version") + ", Level = EventLevel." + eventNode.getAttribute("level")[4:])
+    writeOutput(outputFile, "[Event("+ eventNode.getAttribute("value") + ", Version = " + eventNode.getAttribute("version") + ", Level = EventLevel." + evtLevel)
 
     # Not all events have keywords specified, and some have multiple keywords specified.
     keywords = eventNode.getAttribute("keywords")
     if keywords:
         if " " not in keywords:
             outputFile.write(", Keywords = Keywords." + keywords)
+            evtKeywords = "Keywords." + keywords
         else:
             keywords = keywords.split()
             outputFile.write(", Keywords = ")
             for keywordIndex in range(len(keywords)):
-                outputFile.write("Keywords." + keywords[keywordIndex])
+                evtKeywords += "Keywords." + keywords[keywordIndex]
                 if keywordIndex < (len(keywords) - 1):
-                    outputFile.write(" | ")
-
+                    evtKeywords += " | " 
+            outputFile.write(evtKeywords)
     outputFile.write(")]\n")
 
     # Get the template for the event.
@@ -147,28 +155,10 @@ def generateEvent(eventNode, providerNode, outputFile, stringTable):
     outputFile.write(")\n")
     writeOutput(outputFile, "{\n")
 
-    # Write the call to WriteEvent.
     increaseTabLevel()
-    writeOutput(outputFile, "WriteEvent(" + eventNode.getAttribute("value"))
-
-    # Add method parameters.
-    if argumentCount > 0:
-        # A ',' is needed after the event id.
-        outputFile.write(", ")
-
-        # Write the parameter names to the method call.
-        argumentsProcessed = 0
-        argumentNodes = templateNode.getElementsByTagName("data")
-        for argumentIndex in range(argumentCount):
-            argumentNode = argumentNodes[argumentIndex]
-            argumentName = argumentNode.getAttribute("name")
-            outputFile.write(argumentName)
-            if argumentIndex < (argumentCount - 1):
-                outputFile.write(", ")
-
-    outputFile.write(");\n")
+    writeOutput(outputFile, "// To have this event be emitted from managed side, refer to NativeRuntimeEventSource.cs\n")
+    writeOutput(outputFile, "throw new NotImplementedException();\n")
     decreaseTabLevel()
-
     writeOutput(outputFile, "}\n\n")
 
 

src/coreclr/vm/corelib.cpp

@@ -76,14 +76,15 @@
 #endif
 
 #if defined(FEATURE_EVENTSOURCE_XPLAT)
-#include "nativeeventsource.h"
 #include "eventpipeadapter.h"
 #include "eventpipeinternal.h"
+#include "nativeeventsource.h"
 #endif //defined(FEATURE_EVENTSOURCE_XPLAT)
 
 #ifdef FEATURE_PERFTRACING
 #include "eventpipeadapter.h"
 #include "eventpipeinternal.h"
+#include "nativeeventsource.h"
 #endif //FEATURE_PERFTRACING
 
 #include "tailcallhelp.h"

src/coreclr/vm/ecalllist.h

@@ -989,12 +989,25 @@ FCFuncStart(gStreamFuncs)
 FCFuncEnd()
 
 
-#if defined(FEATURE_EVENTSOURCE_XPLAT)
+#if defined(FEATURE_EVENTSOURCE_XPLAT) || defined(FEATURE_PERFTRACING)
 FCFuncStart(gEventLogger)
+#if defined(FEATURE_EVENTSOURCE_XPLAT)
     QCFuncElement("IsEventSourceLoggingEnabled", XplatEventSourceLogger::IsEventSourceLoggingEnabled)
     QCFuncElement("LogEventSource", XplatEventSourceLogger::LogEventSource)
-FCFuncEnd()
 #endif // defined(FEATURE_EVENTSOURCE_XPLAT)
+#if defined(FEATURE_PERFTRACING)
+    QCFuncElement("LogThreadPoolWorkerThreadStart", NativeEventLogger::LogThreadPoolWorkerThreadStart)
+    QCFuncElement("LogThreadPoolWorkerThreadStop", NativeEventLogger::LogThreadPoolWorkerThreadStop)
+    QCFuncElement("LogThreadPoolWorkerThreadWait", NativeEventLogger::LogThreadPoolWorkerThreadWait)
+    QCFuncElement("LogThreadPoolWorkerThreadAdjustmentSample", NativeEventLogger::LogThreadPoolWorkerThreadAdjustmentSample)
+    QCFuncElement("LogThreadPoolWorkerThreadAdjustmentAdjustment", NativeEventLogger::LogThreadPoolWorkerThreadAdjustmentAdjustment)
+    QCFuncElement("LogThreadPoolWorkerThreadAdjustmentStats", NativeEventLogger::LogThreadPoolWorkerThreadAdjustmentStats)
+    QCFuncElement("LogThreadPoolIOEnqueue", NativeEventLogger::LogThreadPoolIOEnqueue)
+    QCFuncElement("LogThreadPoolIODequeue", NativeEventLogger::LogThreadPoolIODequeue)
+    QCFuncElement("LogThreadPoolWorkingThreadCount", NativeEventLogger::LogThreadPoolWorkingThreadCount)
+#endif // defined(FEATURE_PERFTRACING)
+FCFuncEnd()
+#endif // defined(FEATURE_EVENTSOURCE_XPLAT) || defined(FEATURE_PERFTRACING)
 
 #ifdef FEATURE_PERFTRACING
 FCFuncStart(gEventPipeInternalFuncs)
@@ -1200,7 +1213,7 @@ FCClassElement("WeakReference`1", "System", gWeakReferenceOfTFuncs)
 FCClassElement("X86Base", "System.Runtime.Intrinsics.X86", gX86BaseFuncs)
 #endif // defined(TARGET_X86) || defined(TARGET_AMD64)
 
-#if defined(FEATURE_EVENTSOURCE_XPLAT)
+#if defined(FEATURE_EVENTSOURCE_XPLAT) || defined(FEATURE_PERFTRACING)
 FCClassElement("XplatEventLogger", "System.Diagnostics.Tracing", gEventLogger)
 #endif //defined(FEATURE_EVENTSOURCE_XPLAT)
 

src/coreclr/vm/nativeeventsource.cpp

@@ -9,9 +9,10 @@
 // ============================================================================
 
 #include "common.h"
-#if defined(FEATURE_EVENTSOURCE_XPLAT)
 #include "nativeeventsource.h"
 
+#if defined(FEATURE_EVENTSOURCE_XPLAT)
+
 void QCALLTYPE XplatEventSourceLogger::LogEventSource(__in_z int eventID, __in_z LPCWSTR eventName, __in_z LPCWSTR eventSourceName, __in_z LPCWSTR payload)
 {
     QCALL_CONTRACT;
@@ -31,7 +32,105 @@ BOOL QCALLTYPE XplatEventSourceLogger::IsEventSourceLoggingEnabled()
     END_QCALL;
 
     return retVal;
-
 }
 
 #endif //defined(FEATURE_EVENTSOURCE_XPLAT)
+
+#ifdef FEATURE_PERFTRACING
+
+// These are native QCalls that call into corresponding FireEtw* events for events that want to be emitted from the managed
+// side using NativeRuntimeEventSource.
+// You need to add them to src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/XplatEventLogger.cs and
+// change genRuntimeEventSources.py script to not emit the body that throws NotImplementedException for the event that 
+// want to be fired from managed code.
+// See https://github.com/dotnet/runtime/pull/47829 for an example of how to do this.
+void QCALLTYPE NativeEventLogger::LogThreadPoolWorkerThreadStart(__in_z uint activeWorkerThreadCount, __in_z uint retiredWorkerThreadCount, __in_z short clrInstanceID)
+{
+    QCALL_CONTRACT;
+    BEGIN_QCALL;
+
+    FireEtwThreadPoolWorkerThreadStart(activeWorkerThreadCount, retiredWorkerThreadCount, clrInstanceID);
+
+    END_QCALL;
+}
+
+void QCALLTYPE NativeEventLogger::LogThreadPoolWorkerThreadStop(__in_z uint activeWorkerThreadCount, __in_z uint retiredWorkerThreadCount, __in_z short clrInstanceID)
+{
+    QCALL_CONTRACT;
+    BEGIN_QCALL;
+
+    FireEtwThreadPoolWorkerThreadStop(activeWorkerThreadCount, retiredWorkerThreadCount, clrInstanceID);
+
+    END_QCALL;
+}
+
+void QCALLTYPE NativeEventLogger::LogThreadPoolWorkerThreadWait(__in_z uint activeWorkerThreadCount, __in_z uint retiredWorkerThreadCount, __in_z short clrInstanceID)
+{
+    QCALL_CONTRACT;
+    BEGIN_QCALL;
+
+    FireEtwThreadPoolWorkerThreadWait(activeWorkerThreadCount, retiredWorkerThreadCount, clrInstanceID);
+
+    END_QCALL;
+}
+
+void QCALLTYPE NativeEventLogger::LogThreadPoolWorkerThreadAdjustmentSample(__in_z double throughput, __in_z short clrInstanceID)
+{
+    QCALL_CONTRACT;
+    BEGIN_QCALL;
+
+    FireEtwThreadPoolWorkerThreadAdjustmentSample(throughput, clrInstanceID);
+
+    END_QCALL;
+}
+
+void QCALLTYPE NativeEventLogger::LogThreadPoolWorkerThreadAdjustmentAdjustment(__in_z double averageThroughput, __in_z uint newWorkerThreadCount, __in_z uint reason, __in_z short clrInstanceID)
+{
+    QCALL_CONTRACT;
+    BEGIN_QCALL;
+
+    FireEtwThreadPoolWorkerThreadAdjustmentAdjustment(averageThroughput, newWorkerThreadCount, reason, clrInstanceID);
+    
+    END_QCALL;
+}
+
+void QCALLTYPE NativeEventLogger::LogThreadPoolWorkerThreadAdjustmentStats(__in_z double duration, __in_z double throughput, __in_z double threadWave, __in_z double throughputWave, __in_z double throughputErrorEstimate, __in_z double AverageThroughputErrorEstimate, __in_z double ThroughputRatio, __in_z double confidence, __in_z double newControlSetting, __in_z short newThreadWaveMagnitude, __in_z short ClrInstanceID)
+{
+    QCALL_CONTRACT;
+    BEGIN_QCALL;
+
+    FireEtwThreadPoolWorkerThreadAdjustmentStats(duration, throughput, threadWave, throughputWave, throughputErrorEstimate, AverageThroughputErrorEstimate, ThroughputRatio, confidence, newControlSetting, newThreadWaveMagnitude, ClrInstanceID);
+
+    END_QCALL;
+}
+
+void QCALLTYPE NativeEventLogger::LogThreadPoolIOEnqueue(__in_z void* nativeOverlapped, __in_z void* overlapped, __in_z bool multiDequeues, __in_z short ClrInstanceID)
+{
+    QCALL_CONTRACT;
+    BEGIN_QCALL;
+
+    FireEtwThreadPoolIOEnqueue(nativeOverlapped, overlapped, multiDequeues, ClrInstanceID);
+
+    END_QCALL;
+}
+
+void QCALLTYPE NativeEventLogger::LogThreadPoolIODequeue(__in_z void* nativeOverlapped, __in_z void* overlapped, __in_z short ClrInstanceID)
+{
+    QCALL_CONTRACT;
+    BEGIN_QCALL;
+
+    FireEtwThreadPoolIODequeue(nativeOverlapped, overlapped, ClrInstanceID);
+
+    END_QCALL;
+}
+
+void QCALLTYPE NativeEventLogger::LogThreadPoolWorkingThreadCount(__in_z uint count, __in_z short ClrInstanceID)
+{
+    QCALL_CONTRACT;
+    BEGIN_QCALL;
+
+    FireEtwThreadPoolWorkingThreadCount(count, ClrInstanceID);
+
+    END_QCALL;
+}
+#endif // FEATURE_PERFTRACING

src/coreclr/vm/nativeeventsource.h

@@ -12,13 +12,31 @@
 #ifndef _NATIVEEVENTSOURCE_H_
 #define _NATIVEEVENTSOURCE_H_
 
-#if defined(FEATURE_EVENTSOURCE_XPLAT)
+#include "qcall.h"
+
+#if defined(FEATURE_PERFTRACING)
 class XplatEventSourceLogger
 {
 public:
     static void QCALLTYPE LogEventSource(__in_z int eventID, __in_z LPCWSTR eventName, __in_z LPCWSTR eventSourceName, __in_z LPCWSTR payload);
     static BOOL QCALLTYPE IsEventSourceLoggingEnabled();
 };
+#endif //defined(FEATURE_PERFTRACING)
+
+#if defined(FEATURE_PERFTRACING)
+class NativeEventLogger
+{
+public:
+    static void QCALLTYPE LogThreadPoolWorkerThreadStart(__in_z uint activeWorkerThreadCount, __in_z uint retiredWorkerThreadCount, __in_z short clrInstanceID);
+    static void QCALLTYPE LogThreadPoolWorkerThreadStop(__in_z uint activeWorkerThreadCount, __in_z uint retiredWorkerThreadCount, __in_z short clrInstanceID);
+    static void QCALLTYPE LogThreadPoolWorkerThreadWait(__in_z uint activeWorkerThreadCount, __in_z uint retiredWorkerThreadCount, __in_z short clrInstanceID);
+    static void QCALLTYPE LogThreadPoolWorkerThreadAdjustmentSample(__in_z double throughput, __in_z short clrInstanceID);
+    static void QCALLTYPE LogThreadPoolWorkerThreadAdjustmentAdjustment(__in_z double averageThroughput, __in_z uint newWorkerThreadCount, __in_z uint reason, __in_z short clrInstanceID);
+    static void QCALLTYPE LogThreadPoolWorkerThreadAdjustmentStats(__in_z double duration, __in_z double throughput, __in_z double threadWave, __in_z double throughputWave, __in_z double throughputErrorEstimate, __in_z double AverageThroughputErrorEstimate, __in_z double ThroughputRatio, __in_z double confidence, __in_z double newControlSetting, __in_z short newThreadWaveMagnitude, __in_z short ClrInstanceID);
+    static void QCALLTYPE LogThreadPoolIOEnqueue(__in_z void* nativeOverlapped, __in_z void* overlapped, __in_z bool multiDequeues, __in_z short ClrInstanceID);
+    static void QCALLTYPE LogThreadPoolIODequeue(__in_z void* nativeOverlapped, __in_z void* overlapped, __in_z short ClrInstanceID);
+    static void QCALLTYPE LogThreadPoolWorkingThreadCount(__in_z uint count, __in_z short ClrInstanceID);
+};
+#endif // defined(FEATURE_PERFTRACING)
 
-#endif //defined(FEATURE_EVENTSOURCE_XPLAT)
 #endif //_NATIVEEVENTSOURCE_H_

src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems

@@ -1957,7 +1957,7 @@
     <Compile Include="$(MSBuildThisFileDirectory)System\Threading\ThreadPool.Portable.cs" Condition="'$(FeatureCoreCLR)' != 'true'" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Threading\ThreadPoolBoundHandle.PlatformNotSupported.cs" Condition="'$(FeatureCoreCLR)' != 'true'" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Threading\PortableThreadPool.cs" />
-    <Compile Include="$(MSBuildThisFileDirectory)System\Threading\PortableThreadPoolEventSource.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)System\Threading\NativeRuntimeEventSource.PortableThreadPool.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Threading\PortableThreadPool.GateThread.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Threading\PortableThreadPool.HillClimbing.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Threading\PortableThreadPool.HillClimbing.Complex.cs" />

src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventSource.cs

@@ -229,17 +229,18 @@ internal sealed class EventSourceAutoGenerateAttribute : Attribute
     /// </remarks>
     public partial class EventSource : IDisposable
     {
-#if FEATURE_EVENTSOURCE_XPLAT
-#pragma warning disable CA1823 // field is used to keep listener alive
-        private static readonly EventListener? persistent_Xplat_Listener = XplatEventLogger.InitializePersistentListener();
-#pragma warning restore CA1823
-#endif //FEATURE_EVENTSOURCE_XPLAT
 
         internal static bool IsSupported { get; } = InitializeIsSupported();
 
         private static bool InitializeIsSupported() =>
             AppContext.TryGetSwitch("System.Diagnostics.Tracing.EventSource.IsSupported", out bool isSupported) ? isSupported : true;
 
+#if FEATURE_EVENTSOURCE_XPLAT
+#pragma warning disable CA1823 // field is used to keep listener alive
+        private static readonly EventListener? persistent_Xplat_Listener = IsSupported ? XplatEventLogger.InitializePersistentListener() : null;
+#pragma warning restore CA1823
+#endif //FEATURE_EVENTSOURCE_XPLAT
+
         /// <summary>
         /// The human-friendly name of the eventSource.  It defaults to the simple name of the class
         /// </summary>
@@ -3804,11 +3805,10 @@ public event EventHandler<EventSourceCreatedEventArgs>? EventSourceCreated
         static EventListener()
         {
 #if FEATURE_PERFTRACING
-            // Ensure that NativeRuntimeEventSource is initialized so that EventListeners get an opportunity to subscribe to its events.
-            // This is required because NativeRuntimeEventSource never emit events on its own, and thus will never be initialized
-            // in the normal way that EventSources are initialized.
+            // This allows NativeRuntimeEventSource to get initialized so that EventListeners can subscribe to the runtime events emitted from
+            // native side.
             GC.KeepAlive(NativeRuntimeEventSource.Log);
-#endif // FEATURE_PERFTRACING
+#endif
         }
 
         /// <summary>