Output more useful performance metrics from WinMLRunner's perf console output (#154)

* Separated performance metrics between first iteration and iterations after first iteration

* Added checking to make sure iterations > 1 for printing performance results to csv

* Changed readme

* Changed -perf flag, description

* Made change so that it doesn't process 'all'

* Moved i++ to the correct location for commandline args

Author: Ryan Lai

Tools/WinMLRunner/README.md

@@ -22,7 +22,7 @@ Required command-Line arguments:
 -folder <path>           : Fully qualifed path to a folder with .onnx and/or .pb models, will run all of the models in the folder.
 
 #Optional command-line arguments:
--Perf                    : Captures GPU, CPU, and wall-clock time measurements.
+-Perf                    : optional:<all>:  capture performance measurements such as timing and memory usage. Specifying "all" will output all measurements
 -Iterations <int>        : Number of times to evaluate the model when capturing performance measurements.
 -CPU                     : Will create a session on the CPU.
 -GPU                     : Will create a session on the GPU.
@@ -38,7 +38,6 @@ Required command-Line arguments:
 -Tensor                  : Will load the input as a tensor.
 -Input <image/CSV path>  : Will bind image/data from CSV to model.
 -PerfOutput <CSV path>   : Path to the CSV where the perf results will be written.
--IgnoreFirstRun          : Will ignore the first run in the perf results when calculating the average
 -SavePerIterationPerf	 : Save per iteration performance results to csv file.
 -Debug                   : Will start a trace logging session.
 -Terse                   : Will suppress repetitive console output (initial iteration and summary info will be output).

Tools/WinMLRunner/src/CommandLineArgs.cpp

@@ -24,11 +24,10 @@ void CommandLineArgs::PrintUsage() {
     std::cout << "  -RGB : load the input as an RGB image" << std::endl;
     std::cout << "  -BGR : load the input as a BGR image" << std::endl;
     std::cout << "  -Tensor : load the input as a tensor" << std::endl;
-    std::cout << "  -Perf : capture timing measurements" << std::endl;
+    std::cout << "  -Perf optional:<all>: capture performance measurements such as timing and memory usage. Specifying \"all\" will output all measurements" << std::endl;
     std::cout << "  -Iterations : # times perf measurements will be run/averaged" << std::endl;
     std::cout << "  -Input <fully qualified path>: binds image or CSV to model" << std::endl;
     std::cout << "  -PerfOutput optional:<fully qualified path>: csv file to write the perf results to" << std::endl;
-    std::cout << "  -IgnoreFirstRun : ignore the first run in the perf results when calculating the average" << std::endl;
     std::cout << "  -SavePerIterationPerf : save per iteration performance results to csv file" << std::endl;
     std::cout << "  -SaveTensorData <saveMode>: save first iteration or all iteration output tensor results to csv file [First, All]" << std::endl;
     std::cout << "  -Debug: print trace logs" << std::endl;
@@ -119,12 +118,13 @@ CommandLineArgs::CommandLineArgs(const std::vector<std::wstring>& args)
         {
             m_useGPUBoundInput = true;
         }
-        else if ((_wcsicmp(args[i].c_str(), L"-IgnoreFirstRun") == 0))
-        {
-            m_ignoreFirstRun = true;
-        }
         else if ((_wcsicmp(args[i].c_str(), L"-Perf") == 0))
         {
+            if (i + 1 < args.size() && args[i + 1][0] != L'-' && (_wcsicmp(args[i+1].c_str(), L"all") == 0))
+            {
+                m_perfConsoleOutputAll = true;
+                i++;
+            }
             m_perfCapture = true;
         }
         else if ((_wcsicmp(args[i].c_str(), L"-Debug") == 0))

Tools/WinMLRunner/src/CommandLineArgs.h

@@ -11,8 +11,8 @@ class CommandLineArgs
     bool IsUsingGPUMinPower() const { return m_useGPUMinPower; }
     bool UseBGR() const { return m_useBGR; }
     bool IsUsingGPUBoundInput() const { return m_useGPUBoundInput; }
-    bool IsIgnoreFirstRun() const { return m_ignoreFirstRun; }
     bool IsPerformanceCapture() const { return m_perfCapture; }
+    bool IsPerformanceConsoleOutputVerbose() const { return m_perfConsoleOutputAll; }
     bool IsDebugOutputEnabled() const { return m_debug; }
     bool TerseOutput() const { return m_terseOutput; }
     bool IsPerIterationCapture() const { return m_perIterCapture; }
@@ -107,6 +107,7 @@ class CommandLineArgs
 
 private:
     bool m_perfCapture = false;
+    bool m_perfConsoleOutputAll = false;
     bool m_useCPU = false;
     bool m_useGPU = false;
     bool m_useGPUHighPerformance = false;

Tools/WinMLRunner/src/Common.h

@@ -35,19 +35,11 @@ enum WINML_MODEL_TEST_PERF
 	CREATE_SESSION,
 	BIND_VALUE,
 	EVAL_MODEL,
+    BIND_VALUE_FIRST_RUN,
 	EVAL_MODEL_FIRST_RUN,
 	COUNT
 };
 
-static std::vector<std::wstring> WINML_MODEL_TEST_PERF_NAMES =
-{
-	L"ENTIRE TEST          ",
-	L"  LOAD MODEL         ",
-	L"  CREATE SESSION     ",
-	L"  BIND VALUE         ",
-	L"  EVAL MODEL         ",
-};
-
 #define MAX_PROFILING_LOOP 100
 
 using namespace winrt;