Add time limit stop iterating for winmlrunner static lib (#246)

Author: ryanlai2

Tools/WinMLRunner/src/CommandLineArgs.h

@@ -21,7 +21,7 @@ class CommandLineArgs
     bool IsAutoScale() const { return m_autoScale; }
     bool IsOutputPerf() const { return m_perfOutput; }
     bool IsSaveTensor() const { return m_saveTensor; }
-
+    bool IsTimeLimitIterations() const { return m_timeLimitIterations; }
     BitmapInterpolationMode AutoScaleInterpMode() const { return m_autoScaleInterpMode; }
 
     const std::vector<std::wstring>& ImagePaths() const { return m_imagePaths; }
@@ -76,6 +76,7 @@ class CommandLineArgs
     bool IsImageInput() const { return !m_imagePaths.empty() && m_csvData.empty(); }
 
     uint32_t NumIterations() const { return m_numIterations; }
+    double IterationTimeLimit() const { return m_iterationTimeLimitMilliseconds; }
     uint32_t NumThreads() const { return m_numThreads; }
     uint32_t ThreadInterval() const { return m_threadInterval; } // Thread interval in milliseconds
     uint32_t TopK() const { return m_topK; }
@@ -114,6 +115,12 @@ class CommandLineArgs
     {
         m_perfFileMetadata.push_back(std::make_pair(key, value));
     }
+    // Stop iterating when total time of iterations after the first iteration exceeds time limit.
+    void SetIterationTimeLimit(const double milliseconds)
+    {
+        m_timeLimitIterations = true;
+        m_iterationTimeLimitMilliseconds = milliseconds;
+    }
     std::wstring SaveTensorMode() const { return m_saveTensorMode; }
 
 private:
@@ -139,6 +146,7 @@ class CommandLineArgs
     bool m_perfOutput = false;
     BitmapInterpolationMode m_autoScaleInterpMode = BitmapInterpolationMode::Cubic;
     bool m_saveTensor = false;
+    bool m_timeLimitIterations = false;
     std::wstring m_saveTensorMode = L"First";
 
     std::wstring m_modelFolderPath;
@@ -153,6 +161,7 @@ class CommandLineArgs
     std::wstring m_perfOutputPath;
     std::wstring m_perIterationDataPath;
     uint32_t m_numIterations = 1;
+    double m_iterationTimeLimitMilliseconds = 0;
     uint32_t m_numThreads = 1;
     uint32_t m_threadInterval = 0;
     uint32_t m_topK = 1;

Tools/WinMLRunner/src/Run.cpp

@@ -660,43 +660,59 @@ void RunConfiguration(CommandLineArgs& args, OutputHelper& output, LearningModel
                       const DeviceCreationLocation deviceCreationLocation, Profiler<WINML_MODEL_TEST_PERF>& profiler,
                       const std::wstring& modelPath, const std::wstring& imagePath)
 {
-    for (uint32_t i = 0; i < args.NumIterations(); i++)
+    Timer iterationTimer;
+    uint32_t iterationNum = 0;
+    for (; iterationNum < args.NumIterations(); iterationNum++)
     {
 #if defined(_AMD64_)
         // PIX markers only work on AMD64
         // If PIX tool was attached then capture already began for the first iteration before
         // session creation. This is to begin PIX capture for each iteration after the first
         // iteration.
-        if (i > 0)
+        if (iterationNum > 0)
         {
             StartPIXCapture(output);
         }
 #endif
+        if (args.IsTimeLimitIterations())
+        {
+            if (iterationNum == 1)
+            {
+                iterationTimer.Start();
+            }
+            else if (iterationNum >= 1 && iterationTimer.Stop() >= args.IterationTimeLimit())
+            {
+                std::cout << "Iteration time exceeded limit specified. Exiting.." << std::endl;
+                break;
+            }
+        }
         LearningModelBinding context(session);
         lastHr = BindInputs(context, model, session, output, deviceType, args, inputBindingType, inputDataType,
-                            winrtDevice, deviceCreationLocation, i, profiler, imagePath);
+                            winrtDevice, deviceCreationLocation, iterationNum, profiler, imagePath);
         if (FAILED(lastHr))
         {
             break;
         }
         LearningModelEvaluationResult result = nullptr;
         bool capture_perf = args.IsPerformanceCapture() || args.IsPerIterationCapture();
-        lastHr = EvaluateModel(result, model, context, session, args, output, capture_perf, i, profiler);
+        lastHr = EvaluateModel(result, model, context, session, args, output, capture_perf, iterationNum, profiler);
         if (FAILED(lastHr))
         {
-            output.PrintEvaluatingInfo(i + 1, deviceType, inputBindingType, inputDataType, deviceCreationLocation,
+            output.PrintEvaluatingInfo(iterationNum + 1, deviceType, inputBindingType, inputDataType,
+                                       deviceCreationLocation,
                                        "[FAILED]");
             break;
         }
-        else if (!args.TerseOutput() || i == 0)
+        else if (!args.TerseOutput() || iterationNum == 0)
         {
-            output.PrintEvaluatingInfo(i + 1, deviceType, inputBindingType, inputDataType, deviceCreationLocation,
+            output.PrintEvaluatingInfo(iterationNum + 1, deviceType, inputBindingType, inputDataType,
+                                       deviceCreationLocation,
                                        "[SUCCESS]");
 
             // Only print eval results on the first iteration, iff it's not garbage data
             if (!args.IsGarbageInput() || args.IsSaveTensor())
             {
-                BindingUtilities::PrintOrSaveEvaluationResults(model, args, result.Outputs(), output, i);
+                BindingUtilities::PrintOrSaveEvaluationResults(model, args, result.Outputs(), output, iterationNum);
             }
 
             if (args.TerseOutput() && args.NumIterations() > 1)
@@ -713,15 +729,15 @@ void RunConfiguration(CommandLineArgs& args, OutputHelper& output, LearningModel
     // print metrics after iterations
     if (SUCCEEDED(lastHr) && args.IsPerformanceCapture())
     {
-        output.PrintResults(profiler, args.NumIterations(), deviceType, inputBindingType, inputDataType,
+        output.PrintResults(profiler, iterationNum, deviceType, inputBindingType, inputDataType,
                             deviceCreationLocation, args.IsPerformanceConsoleOutputVerbose());
         if (args.IsOutputPerf())
         {
             std::string deviceTypeStringified = TypeHelper::Stringify(deviceType);
             std::string inputDataTypeStringified = TypeHelper::Stringify(inputDataType);
             std::string inputBindingTypeStringified = TypeHelper::Stringify(inputBindingType);
             std::string deviceCreationLocationStringified = TypeHelper::Stringify(deviceCreationLocation);
-            output.WritePerformanceDataToCSV(profiler, args.NumIterations(), modelPath, deviceTypeStringified,
+            output.WritePerformanceDataToCSV(profiler, iterationNum, modelPath, deviceTypeStringified,
                                                 inputDataTypeStringified, inputBindingTypeStringified,
                                                 deviceCreationLocationStringified, args.GetPerformanceFileMetadata());
         }