com.unity.test-framework.performance@0.1.42-preview

## [0.1.42] - 2018-10-15

### Improvements to report window and minor fixes

Save profiler output on perf tests
Add a button on report window to open profiler output for test
Remove unsupported features for legacy scripting runtime
Fix version attribute for test cases
Remove unnecessary assembly definition

## [0.1.41] - 2018-10-2

### Test report graph

Author: Unity Technologies

CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## [0.1.42] - 2018-10-15
+
+### Improvements to report window and minor fixes
+
+Save profiler output on perf tests
+Add a button on report window to open profiler output for test
+Remove unsupported features for legacy scripting runtime
+Fix version attribute for test cases
+Remove unnecessary assembly definition
+
 ## [0.1.41] - 2018-10-2
 
 ### Test report graph

Documentation~/images/graphtool2.png

@@ -289,6 +289,19 @@ The Performance Test Report is split into two views: the *test view* and the *sa
 
 ![Performance test report](images/graphtool.png)
 
+## Viewing profiler data
+
+When running performance tests on editor or standalone platforms, the framework saves profiler data locally for each test. To view profiler data for a test:
+1. Open Profiler window
+2. Open Performance Test Report window
+3. Select the test you wish to view profiler data
+4. Click on "Load profiler data for test"
+5. Go back to profiler window, the data should be loaded
+
+*Note: If the button is not visible, it means profiler data has not been saved for the test.*
+
+![Open profiler button](images/graphtool2.png)
+
 ## More Examples
 
 #### Example 1: Measure execution time to serialize simple object to JSON

Documentation~/index.md

@@ -13,22 +13,27 @@ class TestListTableItem : TreeViewItem
         public PerformanceTest test;
         public double deviation;
         public double standardDeviation;
+        public double median;
 
-        public TestListTableItem(int id, int depth, string displayName, PerformanceTest test) : base(id, depth, displayName)
+        public TestListTableItem(int id, int depth, string displayName, PerformanceTest test) : base(id, depth,
+            displayName)
         {
             this.test = test;
 
             deviation = 0f;
-            if (test!=null)
+            if (test != null)
             {
                 foreach (var sample in test.SampleGroups)
                 {
                     if (sample.StandardDeviation > deviation)
                         standardDeviation = sample.StandardDeviation;
-                    
+
                     double thisDeviation = sample.StandardDeviation / sample.Median;
                     if (thisDeviation > deviation)
+                    {
                         deviation = thisDeviation;
+                        median = sample.Median;
+                    }
                 }
             }
         }
@@ -48,6 +53,7 @@ public enum MyColumns
             SampleCount,
             StandardDeviation,
             Deviation,
+            Median
         }
 
         public enum SortOption
@@ -56,6 +62,7 @@ public enum SortOption
             SampleCount,
             StandardDeviation,
             Deviation,
+            Median
         }
 
         // Sort options per column
@@ -65,20 +72,25 @@ public enum SortOption
             SortOption.SampleCount,
             SortOption.StandardDeviation,
             SortOption.Deviation,
+            SortOption.Median
         };
 
 
-        public TestListTable(TreeViewState state, MultiColumnHeader multicolumnHeader, TestReportWindow testReportWindow) : base(state, multicolumnHeader)
+        public TestListTable(TreeViewState state, MultiColumnHeader multicolumnHeader,
+            TestReportWindow testReportWindow) : base(state, multicolumnHeader)
         {
             m_testReportWindow = testReportWindow;
 
-            Assert.AreEqual(m_SortOptions.Length, Enum.GetValues(typeof(MyColumns)).Length, "Ensure number of sort options are in sync with number of MyColumns enum values");
+            Assert.AreEqual(m_SortOptions.Length, Enum.GetValues(typeof(MyColumns)).Length,
+                "Ensure number of sort options are in sync with number of MyColumns enum values");
 
             // Custom setup
             rowHeight = kRowHeights;
             showAlternatingRowBackgrounds = true;
             showBorder = true;
-            customFoldoutYOffset = (kRowHeights - EditorGUIUtility.singleLineHeight) * 0.5f; // center foldout in the row since we also center content. See RowGUI
+            customFoldoutYOffset =
+                (kRowHeights - EditorGUIUtility.singleLineHeight) *
+                0.5f; // center foldout in the row since we also center content. See RowGUI
             // extraSpaceBeforeIconAndLabel = 0;
             multicolumnHeader.sortingChanged += OnSortingChanged;
 
@@ -186,6 +198,9 @@ void SortByMultipleColumns()
                     case SortOption.StandardDeviation:
                         orderedQuery = orderedQuery.ThenBy(l => l.standardDeviation, ascending);
                         break;
+                    case SortOption.Median:
+                        orderedQuery = orderedQuery.ThenBy(l => l.median, ascending);
+                        break;
                 }
             }
 
@@ -206,6 +221,8 @@ IOrderedEnumerable<TestListTableItem> InitialOrder(IEnumerable<TestListTableItem
                     return myTypes.Order(l => l.deviation, ascending);
                 case SortOption.StandardDeviation:
                     return myTypes.Order(l => l.standardDeviation, ascending);
+                case SortOption.Median:
+                    return myTypes.Order(l => l.median, ascending);
                 default:
                     Assert.IsTrue(false, "Unhandled enum");
                     break;
@@ -217,11 +234,11 @@ IOrderedEnumerable<TestListTableItem> InitialOrder(IEnumerable<TestListTableItem
 
         protected override void RowGUI(RowGUIArgs args)
         {
-            var item = (TestListTableItem)args.item;
+            var item = (TestListTableItem) args.item;
 
             for (int i = 0; i < args.GetNumVisibleColumns(); ++i)
             {
-                CellGUI(args.GetCellRect(i), item, (MyColumns)args.GetColumn(i), ref args);
+                CellGUI(args.GetCellRect(i), item, (MyColumns) args.GetColumn(i), ref args);
             }
         }
 
@@ -233,10 +250,10 @@ void CellGUI(Rect cellRect, TestListTableItem item, MyColumns column, ref RowGUI
             switch (column)
             {
                 case MyColumns.Name:
-                    {
-                        args.rowRect = cellRect;
-                        base.RowGUI(args);
-                    }
+                {
+                    args.rowRect = cellRect;
+                    base.RowGUI(args);
+                }
                     break;
                 case MyColumns.SampleCount:
                     EditorGUI.LabelField(cellRect, string.Format("{0}", item.test.SampleGroups.Count));
@@ -247,6 +264,9 @@ void CellGUI(Rect cellRect, TestListTableItem item, MyColumns column, ref RowGUI
                 case MyColumns.StandardDeviation:
                     EditorGUI.LabelField(cellRect, string.Format("{0:f2}", item.standardDeviation));
                     break;
+                case MyColumns.Median:
+                    EditorGUI.LabelField(cellRect, string.Format("{0:f2}", item.median));
+                    break;
             }
         }
 
@@ -273,7 +293,7 @@ public static MultiColumnHeaderState CreateDefaultMultiColumnHeaderState(float t
                 autoResize = false,
                 allowToggleVisibility = false
             });
-            string[] names = { "Sample Groups", "Standard Deviation", "Deviation" };
+            string[] names = {"Sample Groups", "Standard Deviation", "Deviation", "Median"};
             foreach (var name in names)
             {
                 var column = new MultiColumnHeaderState.Column
@@ -287,18 +307,23 @@ public static MultiColumnHeaderState CreateDefaultMultiColumnHeaderState(float t
                     autoResize = true
                 };
                 columnList.Add(column);
-            };
+            }
+
+            ;
             var columns = columnList.ToArray();
 
-            Assert.AreEqual(columns.Length, Enum.GetValues(typeof(MyColumns)).Length, "Number of columns should match number of enum values: You probably forgot to update one of them.");
+            Assert.AreEqual(columns.Length, Enum.GetValues(typeof(MyColumns)).Length,
+                "Number of columns should match number of enum values: You probably forgot to update one of them.");
 
             var state = new MultiColumnHeaderState(columns);
-            state.visibleColumns = new int[] {
-                        (int)MyColumns.Name,
-                        (int)MyColumns.SampleCount,
-                        (int)MyColumns.Deviation,
-                        (int)MyColumns.StandardDeviation
-                    };
+            state.visibleColumns = new int[]
+            {
+                (int) MyColumns.Name,
+                (int) MyColumns.SampleCount,
+                (int) MyColumns.Deviation,
+                (int) MyColumns.StandardDeviation,
+                (int) MyColumns.Median
+            };
             return state;
         }
 
@@ -313,7 +338,8 @@ protected override void SelectionChanged(IList<int> selectedIds)
 
     static class MyExtensionMethods
     {
-        public static IOrderedEnumerable<T> Order<T, TKey>(this IEnumerable<T> source, Func<T, TKey> selector, bool ascending)
+        public static IOrderedEnumerable<T> Order<T, TKey>(this IEnumerable<T> source, Func<T, TKey> selector,
+            bool ascending)
         {
             if (ascending)
             {
@@ -325,7 +351,8 @@ public static IOrderedEnumerable<T> Order<T, TKey>(this IEnumerable<T> source, F
             }
         }
 
-        public static IOrderedEnumerable<T> ThenBy<T, TKey>(this IOrderedEnumerable<T> source, Func<T, TKey> selector, bool ascending)
+        public static IOrderedEnumerable<T> ThenBy<T, TKey>(this IOrderedEnumerable<T> source, Func<T, TKey> selector,
+            bool ascending)
         {
             if (ascending)
             {
@@ -337,4 +364,4 @@ public static IOrderedEnumerable<T> ThenBy<T, TKey>(this IOrderedEnumerable<T> s
             }
         }
     }
-}
+}

Editor/TestReportGraph/Gintas.TestRunner.Graph.asmdef

@@ -8,6 +8,7 @@
 using System.IO;
 using System.Linq;
 using Unity.PerformanceTesting;
+using Unity.PerformanceTesting.Runtime;
 
 namespace Unity.PerformanceTesting
 {
@@ -25,6 +26,7 @@ public class TestReportWindow : EditorWindow
         public Color m_colorStandardLine = new Color(1.0f, 1.0f, 1.0f);
         public Color m_colorMedianLine = new Color(0.2f, 0.5f, 1.0f, 0.5f);
         public Color m_colorMedianText = new Color(0.4f, 0.7f, 1.0f, 1.0f);
+        public Color m_colorWarningText = Color.red;
         private PerformanceTestRun m_resultsData;
         private string m_selectedTest;
 
@@ -177,10 +179,6 @@ private void OnGUI()
             Draw();
         }
 
-        private void Update()
-        {
-        }
-
         private double GetPercentageOffset(List<SamplePoint> samplePoint, float percent, out int outputFrameIndex)
         {
             int index = (int) ((samplePoint.Count - 1) * percent / 100);
@@ -240,9 +238,9 @@ private void Draw()
             {
                 LoadData();
                 CreateTestListTable();
-                if(m_resultsData == null) return;
-                if(m_resultsData.Results == null) return;
-                if(m_resultsData.Results.Any(result => result.TestName == m_selectedTest))
+                if (m_resultsData == null) return;
+                if (m_resultsData.Results == null) return;
+                if (m_resultsData.Results.Any(result => result.TestName == m_selectedTest))
                     SelectTest(m_selectedTest);
                 else
                     SelectTest(0);
@@ -251,7 +249,7 @@ private void Draw()
             if (m_resultsData == null)
                 return;
 
-            if (m_resultsData.Results.Count <=0)
+            if (m_resultsData.Results.Count <= 0)
                 GUILayout.Label("No performance test data found.\nNote this is supported only on 2018.3 or newer.");
             else
             {
@@ -266,6 +264,19 @@ private void Draw()
                     }
                 }
 
+                if (!string.IsNullOrEmpty(m_selectedTest))
+                {
+                    var profileDataFile = Path.Combine(Application.persistentDataPath,
+                        Utils.RemoveIllegalCharacters(m_selectedTest) + ".raw");
+                    if (File.Exists(profileDataFile))
+                    {
+                        if (GUILayout.Button(string.Format("Load profiler data for test: {0}", m_selectedTest)))
+                        {
+                            ProfilerDriver.LoadProfile(profileDataFile, false);
+                        }
+                    }
+                }
+
                 m_showSamples = BoldFoldout(m_showSamples, "Sample Group View");
                 if (m_showSamples)
                 {
@@ -283,6 +294,7 @@ private void Draw()
                             dataIndex += result.SampleGroups.Count;
                             continue;
                         }
+
                         foreach (var sampleGroup in result.SampleGroups)
                         {
                             var data = m_sampleGroupAdditionalData[dataIndex];
@@ -308,7 +320,10 @@ private void Draw()
                             Draw2Column("Max", max);
                             GUILayout.FlexibleSpace();
                             Color oldColor = GUI.contentColor;
-                            GUI.contentColor = m_colorMedianText;
+                            if (median < 0.01f)
+                                GUI.contentColor = m_colorWarningText;
+                            else
+                                GUI.contentColor = m_colorMedianText;
                             Draw2Column("Median", median);
                             GUI.contentColor = oldColor;
                             //Draw2Column("SD", (float)sampleGroup.StandardDeviation);
@@ -318,7 +333,7 @@ private void Draw()
                             EditorGUILayout.EndVertical();
                             DrawBarGraph(position.width - 200, 100, sampleGroup.Samples, graphMin, max, median);
                             DrawBoxAndWhiskerPlot(50, 100, min, lowerQuartile, median, upperQuartile, max, min, max,
-                                (float)sampleGroup.StandardDeviation, m_colorWhite, m_colorBoxAndWhiskerBackground);
+                                (float) sampleGroup.StandardDeviation, m_colorWhite, m_colorBoxAndWhiskerBackground);
                             EditorGUILayout.EndHorizontal();
 
                             EditorGUILayout.EndVertical();

Editor/TestReportGraph/Gintas.TestRunner.Graph.asmdef.meta

@@ -21,7 +21,7 @@ private void ValidateInput(string resultXmlFileName)
         {
             if (string.IsNullOrEmpty(resultXmlFileName))
             {
-                throw new ArgumentNullException(resultXmlFileName, nameof(resultXmlFileName));
+                throw new ArgumentNullException(resultXmlFileName, resultXmlFileName);
             }
 
             if (!File.Exists(resultXmlFileName))
@@ -171,7 +171,7 @@ private PerformanceTestRun TryDeserializePerformanceTestRunJsonObject(string jso
 
         private string GetJsonFromHashtag(string tag, string line)
         {
-            if (!line.Contains($"##{tag}:")) return null;
+            if (!line.Contains(string.Format("##{0}:",tag))) return null;
             var jsonStart = line.IndexOf('{');
             var openBrackets = 0;
             var stringIndex = jsonStart;

Editor/TestReportGraph/TestListTable.cs

@@ -21,7 +21,7 @@ To install the Performance Testing Extension package
 ``` json
 {
     "dependencies": {
-        "com.unity.test-framework.performance": "0.1.41-preview",
+        "com.unity.test-framework.performance": "0.1.42-preview",
         "com.unity.modules.jsonserialize": "1.0.0",
         "com.unity.modules.unitywebrequest": "1.0.0",
         "com.unity.modules.unityanalytics": "1.0.0",