Add ability to add metadata to performance output file for code that consumes winmlrunner as static lib (#220)

Author: ryanlai2

Tools/WinMLRunner/src/CommandLineArgs.cpp

@@ -345,9 +345,9 @@ CommandLineArgs::CommandLineArgs(const std::vector<std::wstring>& args)
     CheckForInvalidArguments();
 }
 
-void CommandLineArgs::SetupOutputDirectories(const std::wstring &sBaseOutputPath,
-                                             const std::wstring &sPerfOutputPath,
-                                             const std::wstring &sPerIterationDataPath)
+void CommandLineArgs::SetupOutputDirectories(const std::wstring& sBaseOutputPath,
+                                             const std::wstring& sPerfOutputPath,
+                                             const std::wstring& sPerIterationDataPath)
 {
     std::filesystem::path PerfOutputPath(sPerfOutputPath);
     std::filesystem::path BaseOutputPath(sBaseOutputPath);

Tools/WinMLRunner/src/CommandLineArgs.h

@@ -30,6 +30,7 @@ class CommandLineArgs
     const std::wstring& FolderPath() const { return m_modelFolderPath; }
     const std::wstring& ModelPath() const { return m_modelPath; }
     const std::wstring& PerIterationDataPath() const { return m_perIterationDataPath; }
+    std::vector<std::pair<std::string, std::string>>& GetPerformanceFileMetadata() { return m_perfFileMetadata; }
 #ifdef DXCORE_SUPPORTED_BUILD
     const std::wstring& GetGPUAdapterName() const { return m_adapterName; }
 #endif
@@ -109,7 +110,10 @@ class CommandLineArgs
     void SetTopK(unsigned k) { m_topK = k; }
     void SetPerformanceCSVPath(const std::wstring& performanceCSVPath) { m_perfOutputPath = performanceCSVPath; }
     void SetRunIterations(const uint32_t iterations) { m_numIterations = iterations; }
-
+    void AddPerformanceFileMetadata(const std::pair<std::string, std::string>& metadata)
+    {
+        m_perfFileMetadata.push_back(metadata);
+    }
     std::wstring SaveTensorMode() const { return m_saveTensorMode; }
 
 private:
@@ -151,6 +155,7 @@ class CommandLineArgs
     uint32_t m_numThreads = 1;
     uint32_t m_threadInterval = 0;
     uint32_t m_topK = 1;
+    std::vector<std::pair<std::string, std::string>> m_perfFileMetadata;
 
     void CheckNextArgument(const std::vector<std::wstring>& args, UINT i);
     void CheckForInvalidArguments();

Tools/WinMLRunner/src/OutputHelper.h

@@ -742,7 +742,8 @@ class OutputHelper
 
     void WritePerformanceDataToCSV(const Profiler<WINML_MODEL_TEST_PERF>& profiler, int numIterations,
                                    std::wstring model, const std::string& deviceType, const std::string& inputBinding,
-                                   const std::string& inputType, const std::string& deviceCreationLocation) const
+                                   const std::string& inputType, const std::string& deviceCreationLocation,
+                                   const std::vector<std::pair<std::string,std::string>>& perfFileMetadata) const
     {
         double loadTime = profiler[LOAD_MODEL].GetAverage(CounterType::TIMER);
         double createSessionTime = profiler[CREATE_SESSION].GetAverage(CounterType::TIMER);
@@ -913,7 +914,13 @@ class OutputHelper
                      << ","
                      << "evaluate max shared memory (MB)"
                      << ","
-                     << "evaluate min shared memory (MB)" << std::endl;
+                     << "evaluate min shared memory (MB)"
+                     << ",";
+                for (auto metaDataPair : perfFileMetadata)
+                {
+                    fout << metaDataPair.first << ",";
+                }
+                    fout << std::endl;
             }
             fout << modelName << "," << deviceType << "," << inputBinding << "," << inputType << ","
                  << deviceCreationLocation << "," << numIterations << "," << loadTime << "," << createSessionTime << ","
@@ -941,7 +948,12 @@ class OutputHelper
                  << (numIterations <= 1 ? 0 : minBindSharedMemoryUsage) << "," << firstEvalSharedMemoryUsage << ","
                  << (numIterations <= 1 ? 0 : averageEvalSharedMemoryUsage) << ","
                  << (numIterations <= 1 ? 0 : maxEvalSharedMemoryUsage) << ","
-                 << (numIterations <= 1 ? 0 : minEvalSharedMemoryUsage) << "," << std::endl;
+                 << (numIterations <= 1 ? 0 : minEvalSharedMemoryUsage) << ",";
+            for (auto metaDataPair : perfFileMetadata)
+            {
+                fout << metaDataPair.second << ",";
+            }
+            fout << std::endl;
             fout.close();
         }
     }

Tools/WinMLRunner/src/Run.cpp

@@ -709,7 +709,8 @@ int run(CommandLineArgs& args, Profiler<WINML_MODEL_TEST_PERF>& profiler) try
                                     output.WritePerformanceDataToCSV(profiler, args.NumIterations(), path,
                                                                      deviceTypeStringified, inputDataTypeStringified,
                                                                      inputBindingTypeStringified,
-                                                                     deviceCreationLocationStringified);
+                                                                     deviceCreationLocationStringified,
+                                                                     args.GetPerformanceFileMetadata());
                                 }
                             }