Implement Nettrace Support for Traces with Universal Providers (#2182)

* Implement nettrace conversion to TraceLog using universal providers.

* Add CPU stacks.

* Uncomment assert.

* Multi-process Support

* Clean-up nettrace file views based on what data is present.

Update Nettrace Universal converter with v6 fields.

* React to changes in the Universal providers.

Integral types --> VarInt
Null-terminated UTF16 strings --> Counted UTF8 strings

* Clean-up parsers.

* Clean-up

* Add documentation for the universal providers.

* Updates to universal provider documentation.

* Fix typo

Author: brianrob

src/MemoryGraph/graph.cs

@@ -1070,7 +1070,7 @@ public class Module : IFastSerializable
         /// <summary>
         /// The size of the image when loaded in memory
         /// </summary>
-        public int Size;
+        public long Size;
         /// <summary>
         /// The time when this image was built (There is a field in the PE header).   May be MinimumValue if unknonwn. 
         /// </summary>

src/PerfView/PerfViewData.cs

@@ -42,6 +42,7 @@
 using Utilities;
 using Address = System.UInt64;
 using EventSource = EventSources.EventSource;
+using Microsoft.Diagnostics.Tracing.Parsers.Universal.Events;
 
 namespace PerfView
 {
@@ -6017,7 +6018,7 @@ protected internal override StackSource OpenStackSourceImpl(string streamName, T
                                     {
                                         const int numBuckets = 20;
                                         int bucket = (int)(normalizeDistance * numBuckets);
-                                        int bucketSizeInPages = module.ModuleFile.ImageSize / (numBuckets * 4096);
+                                        int bucketSizeInPages = (int)(module.ModuleFile.ImageSize / (numBuckets * 4096));
                                         string bucketName = "Image Bucket " + bucket + " Size " + bucketSizeInPages + " Pages";
                                         stackIndex = stackSource.Interner.CallStackIntern(stackSource.Interner.FrameIntern(bucketName), stackIndex);
                                     }
@@ -9403,6 +9404,29 @@ public partial class EventPipePerfViewData : PerfViewFile
 
         private string m_extraTopStats;
 
+        public override bool SupportsProcesses => m_supportsProcesses;
+
+        private bool m_supportsProcesses;
+
+        public override List<IProcess> GetProcesses(TextWriter log)
+        {
+            var eventLog = GetTraceLog(log);
+            var processes = new List<IProcess>(eventLog.Processes.Count);
+            foreach (var process in eventLog.Processes)
+            {
+                var iprocess = new IProcessForStackSource(process.Name);
+                iprocess.StartTime = process.StartTime;
+                iprocess.EndTime = process.EndTime;
+                iprocess.CPUTimeMSec = process.CPUMSec;
+                iprocess.ParentID = process.ParentID;
+                iprocess.CommandLine = process.CommandLine;
+                iprocess.ProcessID = process.ProcessID;
+                processes.Add(iprocess);
+            }
+            processes.Sort();
+            return processes;
+        }
+
         protected internal override EventSource OpenEventSourceImpl(TextWriter log)
         {
             var traceLog = GetTraceLog(log);
@@ -9433,6 +9457,8 @@ protected override Action<Action> OpenImpl(Window parentWindow, StatusBar worker
             bool hasAspNetCoreHosting = false;
             bool hasContention = false;
             bool hasWaitHandle = false;
+            bool hasUniversalSystem = false;
+            bool hasUniversalCPU = false;
             if (m_traceLog != null)
             {
                 foreach (TraceEventCounts eventStats in m_traceLog.Stats)
@@ -9486,6 +9512,16 @@ protected override Action<Action> OpenImpl(Window parentWindow, StatusBar worker
                     {
                         hasWaitHandle = true;
                     }
+                    else if (eventStats.ProviderGuid == UniversalSystemTraceEventParser.ProviderGuid)
+                    {
+                        hasUniversalSystem = true;
+                        m_supportsProcesses = true;
+                    }
+                    else if (eventStats.ProviderGuid == UniversalEventsTraceEventParser.ProviderGuid && eventStats.EventName.StartsWith("cpu"))
+                    {
+                        hasUniversalCPU = true;
+                        m_supportsProcesses = true;
+                    }
                 }
             }
 
@@ -9495,13 +9531,30 @@ protected override Action<Action> OpenImpl(Window parentWindow, StatusBar worker
 
             if (m_traceLog != null)
             {
+                if (hasUniversalSystem) // universal
+                {
+                    m_Children.Add(new PerfViewProcesses(this));
+
+                    if (hasUniversalCPU)
+                    {
+                        m_Children.Add(new PerfViewStackSource(this, "CPU"));
+                    }
+                }
+                else // dotnet-trace
+                {
+                    if (hasAnyStacks)
+                    {
+                        m_Children.Add(new PerfViewStackSource(this, "Thread Time (with StartStop Activities)"));
+                    }
+                }
+
+                // Source agnostic
                 m_Children.Add(new PerfViewEventSource(this));
-                m_Children.Add(new PerfViewEventStats(this));
+                advanced.AddChild(new PerfViewEventStats(this));
 
                 if (hasAnyStacks)
                 {
-                    m_Children.Add(new PerfViewStackSource(this, "Thread Time (with StartStop Activities)"));
-                    m_Children.Add(new PerfViewStackSource(this, "Any"));
+                    advanced.AddChild(new PerfViewStackSource(this, "Any"));
                 }
 
                 if (hasGC)
@@ -9750,6 +9803,26 @@ protected internal override StackSource OpenStackSourceImpl(string streamName, T
                         eventSource.Process();
                         stackSource.DoneAddingSamples();
 
+                        return stackSource;
+                    }
+                case "CPU":
+                    {
+                        var eventLog = GetTraceLog(log);
+                        var eventSource = eventLog.Events.GetSource();
+                        var stackSource = new MutableTraceEventStackSource(eventLog);
+                        var sample = new StackSourceSample(stackSource);
+
+                        var universalEventsParser = new UniversalEventsTraceEventParser(eventSource);
+                        universalEventsParser.cpu += delegate (SampleTraceData data)
+                        {
+                            sample.TimeRelativeMSec = data.TimeStampRelativeMSec;
+                            sample.Metric = data.Value;
+                            sample.StackIndex = stackSource.GetCallStack(data.CallStackIndex(), data);
+                            stackSource.AddSample(sample);
+                            
+                        };
+                        eventSource.Process();
+
                         return stackSource;
                     }
                 default:

src/TraceEvent/EventPipe/UniversalProviders.md

@@ -0,0 +1,78 @@
+# Universal Providers Tracing Events
+
+## Overview
+This document describes the definitions of a set of universal machine-wide tracing events that are consumed by TraceEvent and exposed in PerfView.
+
+## Format Restrictions
+ - Any format that can be converted to a TraceLog (ETLX) by TraceEvent can use these event definitions.  At this time, PerfView will only consume them from nettrace files.
+ - Formats must provide a common set of fields with each event.  These fields are used by in addition to the payloads listed in this document to interpret the events.  The fields do not explicitly require data types.  If they don't match what TraceEvent uses for storage, then TraceEvent will need to convert them.  It is recommended to use the types specified in the field descriptions below (if specified).
+ - Formats must provide the QPC frequency to ensure that times that are stored in the Value fields of the Universal Events Provider can be interpreted.
+
+### Required Fields
+ - TimeStamp: The time that the event occurred.
+ - Process ID: The integer identifier of the process that caused the event to be written.
+ - Thread ID: The integer identifier of the thread that caused the event to be written.
+ - Processor ID: The integer identifier of the processor associated with the event.
+
+## Common Format Information
+ - All integers are of type varint encoded in ULEB64/7-bit encoded format.
+ - All strings are length-prefixed (16-bit unsigned integer) followed by UTF8 bytes.  Not null-terminated.
+
+## Universal System Provider
+The universal system provider is named "Universal.System". It's purpose is to provide system-level events for reconstructing the state of a machine and its processes during a period of time.
+
+### Event Definitions
+- **Event Name**: ProcessCreate
+    - **Description**: Identifies a newly created processes. The process ID is inferred from the format's process ID field.
+    - **Field: NamespaceId**: varint - The actual process ID on the system within the running namespace.
+    - **Field: Name**: string - The name of the process.
+    - **Field: NamespaceName**: string - Namespace of the process. Can be used to convey a set of related processes.
+
+- **Event Name**: ProcessExit
+    - **Description**: Identifies a process that has exited. The process ID is inferred from the format's process ID field.
+
+- **Event Name**: ProcessMapping
+    - **Description**: Identifies a mapping (file, memory, etc.) associated with a specific process.
+    - **Field: Id**: varint - A trace-wide unique id for the mapping.  Used to identify this mapping in other events.
+    - **Field: ProcessId**: varint - The process ID on the system that the mapping is for.
+    - **Field: StartAddress**: varint - Starting virtual address of the mapping within the process.
+    - **Field: EndAddress**: varint - Ending virtual address of the mapping within the process.
+    - **Field: FileOffset**: varint - Offset of the actual file on disk that StartAddress correlates to.  For PE files this is always zero.
+    - **Field: FileName**: string - Name of the file the mapping represents.  Empty if none.
+    - **Field: MetadataId**: varint - Identifier for a matcing ProcessMappingMetadataEvent.  0 means no metadata exists for this mapping.
+
+- **Event Name**: ProcessMappingMetadata
+    - **Description**: Contains symbol and version information for the mapping if available.
+    - **Field: Id**: varint - Trace-wide unique identifier for the mapping metadata. Often seen in multiple ProcessMapping events when the same executable file is loaded into multiple processes.
+    - **Field: SymbolMetadata**: string - Symbol metadata in JSON.  Describes the type (ELF/PE) and details that enable looking up symbols locally or remotely.
+    - **Field: VersionMetadata**: string - Version metadata in JSON.  Describes the build and version details.
+
+- **Event Name**: ProcessSymbol
+    - **Description**: Maps a range of address space to a readable name (symbol).
+    - **Field: Id**: varint - Trace-wide unique identifier for the symbol.
+    - **Field: MappingId**: varint - Id of the ProcessMapping that contains the symbol.
+    - **Field: StartAddress**: varint - Starting virtual address of the symbol within the mapping.
+    - **Field: EndAddress**: varint - Ending virtual address of the symbol within the mapping.
+    - **Field: Name**: string - Name of the symbol.
+
+## Universal Events Provider
+The universal events provider is named "Universal.Events". It's purpose is to provide events that contain a value and have a callstack. There are no stable event IDs, but there will be a set of stable names.  Required fields as listed above in this document are used to assist tools in interpreting the events.
+
+### Current Stable Event Names
+ - cpu - Represents a CPU sample.
+ - cswitch - Represents a machine-level context switch.
+
+### Event Definition
+ - **Field: Value**: varint - Value of the event.
+
+### Example: cpu
+ - Each cpu event represents a CPU sample.
+ - The value represents the weight of the sample.
+ - As an example if emitting one CPU sample per core per millisecond, emit an event with Value=1 per core every millisecond.
+
+### Example: cswitch
+ - Each cswitch event represents an OS-level context switch.
+ - The thread ID provided in the set of required fields is the thread being switched in.
+ - The CPU number provided in the set of required fields is the CPU involved.
+ - The value represents the amount of time that that has elapsed since the thread last executed (switched out time).  Times are based on the QPC frequency from the source containing the events.
+ - The callstack associated with the event is the stack where the thread switched out (blocked).

src/TraceEvent/Parsers/UniversalEventsTraceEventParser.cs

@@ -0,0 +1,128 @@
+using System;
+using System.Diagnostics;
+using System.Text;
+using Address = System.UInt64;
+
+#pragma warning disable 1591        // disable warnings on XML comments not being present
+
+namespace Microsoft.Diagnostics.Tracing.Parsers
+{
+    using Microsoft.Diagnostics.Tracing.Parsers.Universal.Events;
+
+    public sealed class UniversalEventsTraceEventParser: TraceEventParser
+    {
+        public static string ProviderName = "Universal.Events";
+        public static Guid ProviderGuid = new Guid("bc5e5d63-9799-5873-33d9-fba8316cef71");
+        public enum Keywords : long
+        {
+            None = 0x0,
+        };
+
+        public UniversalEventsTraceEventParser(TraceEventSource source) : base(source) { }
+
+        public event Action<SampleTraceData> cpu
+        {
+            add
+            {
+                source.RegisterEventTemplate(new SampleTraceData(value, 1, (int)TraceEventTask.Default, "cpu", Guid.Empty, 0, "Default", ProviderGuid, ProviderName));
+            }
+            remove
+            {
+                source.UnregisterEventTemplate(value, 1, Guid.Empty);
+            }
+        }
+
+        public event Action<SampleTraceData> cswitch
+        {
+            add
+            {
+                source.RegisterEventTemplate(new SampleTraceData(value, 2, (int)TraceEventTask.Default, "cswitch", Guid.Empty, 0, "Default", ProviderGuid, ProviderName));
+            }
+            remove
+            {
+                source.UnregisterEventTemplate(value, 2, Guid.Empty);
+            }
+        }
+
+        #region private
+        protected override string GetProviderName() { return ProviderName; }
+
+        static private volatile TraceEvent[] s_templates;
+
+        protected internal override void EnumerateTemplates(Func<string, string, EventFilterResponse> eventsToObserve, Action<TraceEvent> callback)
+        {
+            if (s_templates == null)
+            {
+                var templates = new TraceEvent[2];
+                templates[0] = new SampleTraceData(null, 1, (int)TraceEventTask.Default, "cpu", Guid.Empty, 0, "Default", ProviderGuid, ProviderName);
+                templates[1] = new SampleTraceData(null, 2, (int)TraceEventTask.Default, "cswitch", Guid.Empty, 0, "Default", ProviderGuid, ProviderName);
+                s_templates = templates;
+            }
+            foreach (var template in s_templates)
+                if (eventsToObserve == null || eventsToObserve(template.ProviderName, template.EventName) == EventFilterResponse.AcceptEvent)
+                    callback(template);
+        }
+
+        #endregion
+    }
+}
+
+namespace Microsoft.Diagnostics.Tracing.Parsers.Universal.Events
+{
+    public sealed class SampleTraceData : TraceEvent
+    {
+        public Address Value { get { return GetVarUIntAt(0); } }
+
+        #region Private
+        internal SampleTraceData(Action<SampleTraceData> action, int eventID, int task, string taskName, Guid taskGuid, int opcode, string opcodeName, Guid providerGuid, string providerName)
+            : base(eventID, task, taskName, taskGuid, opcode, opcodeName, providerGuid, providerName)
+        {
+            Action = action;
+        }
+        protected internal override void Dispatch()
+        {
+            Action(this);
+        }
+
+        protected internal override Delegate Target
+        {
+            get { return Action; }
+            set { Action = (Action<SampleTraceData>)value; }
+        }
+        public override StringBuilder ToXml(StringBuilder sb)
+        {
+            Prefix(sb);
+            XmlAttrib(sb, "Value", Value);
+            sb.Append("/>");
+            return sb;
+        }
+
+        public override string[] PayloadNames
+        {
+            get
+            {
+                if (payloadNames == null)
+                    payloadNames = new string[] { "Value" };
+                return payloadNames;
+            }
+        }
+
+        public override object PayloadValue(int index)
+        {
+            switch (index)
+            {
+                case 0:
+                    return Value;
+                default:
+                    Debug.Assert(false, "Bad field index");
+                    return null;
+            }
+        }
+
+        public static ulong GetKeywords() { return 0; }
+        public static string GetProviderName() { return UniversalEventsTraceEventParser.ProviderName; }
+        public static Guid GetProviderGuid() { return UniversalEventsTraceEventParser.ProviderGuid; }
+        private event Action<SampleTraceData> Action;
+        #endregion
+    }
+}