This change allows user to specify folder of images to bind and evaluate (#237)

* Evaluate folder of images

* Evaluate folder of images

* Updated usage

* Test change

* test changes back to test method cleanup

* Fix build break

* remove test

* Added test

* Fix comment typo

* Don't print out image path if using -terse flag

* Remove commitments

Author: Ryan Lai

Testing/WinMLRunnerTest/WinMLRunnerTest.cpp

@@ -641,6 +641,30 @@ namespace WinMLRunnerTest
             // We need to expect one more line because of the header
             Assert::AreEqual(static_cast<size_t>(2), GetOutputCSVLineCount(tensorDataPath + L"\\PerIterationData\\Summary.csv"));
         }
+
+        TEST_METHOD(ProvidedImageInputFolder)
+        {
+            // Make test_folder_input folder before starting the tests
+            std::string mkFolderCommand = "mkdir " + std::string(INPUT_FOLDER_PATH.begin(), INPUT_FOLDER_PATH.end());
+            system(mkFolderCommand.c_str());
+
+            std::vector<std::string> images = { "fish.png", "kitten_224.png" };
+
+            // Copy images from list to test_folder_input
+            for (auto image : images)
+            {
+                std::string copyCommand = "Copy ";
+                copyCommand += image;
+                copyCommand += ' ' + std::string(INPUT_FOLDER_PATH.begin(), INPUT_FOLDER_PATH.end());
+                system(copyCommand.c_str());
+            }
+            const std::wstring command = BuildCommand({ EXE_PATH, L"-model", L"SqueezeNet.onnx", L"-InputImageFolder", INPUT_FOLDER_PATH });
+            Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str()));
+
+            std::string removeCommand = "rd /s /q ";
+            removeCommand += std::string(INPUT_FOLDER_PATH.begin(), INPUT_FOLDER_PATH.end());
+            system(removeCommand.c_str());
+        }
     };
 
     TEST_CLASS(CsvInputTest)

Tools/WinMLRunner/README.md

@@ -45,7 +45,8 @@ Required command-Line arguments:
 -Tensor : load the input as a tensor
 -Perf [all]: capture performance measurements such as timing and memory usage. Specifying "all" will output all measurements
 -Iterations : # times perf measurements will be run/averaged. (maximum: 1024 times)
--Input <fully qualified path>: binds image or CSV to model
+-Input <path to input file>: binds image or CSV to model
+-InputImageFolder <path to directory of images> : specify folder of images to bind to model" << std::endl;
 -TopK <number>: print top <number> values in the result. Default to 1
 -BaseOutputPath [<fully qualified path>] : base output directory path for results, default to cwd
 -PerfOutput [<path>] : fully qualified or relative path including csv filename for perf results

Tools/WinMLRunner/src/BindingUtilities.h

@@ -477,8 +477,8 @@ namespace BindingUtilities
         }
         else if (args.IsImageInput())
         {
-            // Creating Tensors for Input Images haven't been added yet.
-            throw hresult_not_implemented(L"Creating Tensors for Input Images haven't been implemented yet!");
+            // Creating Tensors for Input Images haven't been added.
+            throw hresult_not_implemented(L"Creating Tensors for Input Images haven't been implemented!");
         }
 
         if (inputBindingType == InputBindingType::CPU)
@@ -605,7 +605,7 @@ namespace BindingUtilities
             }
             else
             {
-                throw hresult_not_implemented(L"BitmapPixel format not yet handled by WinMLRunner.");
+                throw hresult_not_implemented(L"BitmapPixel format not handled by WinMLRunner.");
             }
             std::vector<int64_t> shape = { 1, channels, imageFeatureDescriptor.Height(),
                                            imageFeatureDescriptor.Width() };

Tools/WinMLRunner/src/CommandLineArgs.cpp

@@ -5,6 +5,7 @@
 #include <ctime>
 #include <iomanip>
 #include <filesystem>
+#include "Filehelper.h"
 
 using namespace Windows::AI::MachineLearning;
 
@@ -35,7 +36,8 @@ void CommandLineArgs::PrintUsage()
                  "will output all measurements"
               << std::endl;
     std::cout << "  -Iterations : # times perf measurements will be run/averaged. (maximum: 1024 times)" << std::endl;
-    std::cout << "  -Input <fully qualified path> : binds image or CSV to model" << std::endl;
+    std::cout << "  -Input <path to input file>: binds image or CSV to model" << std::endl;
+    std::cout << "  -InputImageFolder <path to directory of images> : specify folder of images to bind to model" << std::endl;
     std::cout << "  -TopK <number> : print top <number> values in the result. Default to 1" << std::endl;
     std::cout << "  -BaseOutputPath [<fully qualified path>] : base output directory path for results, default to cwd"
               << std::endl;
@@ -139,7 +141,13 @@ CommandLineArgs::CommandLineArgs(const std::vector<std::wstring>& args)
         }
         else if ((_wcsicmp(args[i].c_str(), L"-Input") == 0))
         {
-            m_inputData = args[++i];
+            CheckNextArgument(args, i);
+            m_inputData = FileHelper::GetAbsolutePath(args[++i]);
+        }
+        else if ((_wcsicmp(args[i].c_str(), L"-InputImageFolder") == 0))
+        {
+            CheckNextArgument(args, i);
+            m_inputImageFolderPath = FileHelper::GetAbsolutePath(args[++i]);
         }
         else if ((_wcsicmp(args[i].c_str(), L"-PerfOutput") == 0))
         {
@@ -326,7 +334,7 @@ CommandLineArgs::CommandLineArgs(const std::vector<std::wstring>& args)
         if (m_inputData.find(L".png") != std::string::npos || m_inputData.find(L".jpg") != std::string::npos ||
             m_inputData.find(L".jpeg") != std::string::npos)
         {
-            m_imagePath = m_inputData;
+            m_imagePaths.push_back(m_inputData);
         }
         else if (m_inputData.find(L".csv") != std::string::npos)
         {
@@ -339,12 +347,31 @@ CommandLineArgs::CommandLineArgs(const std::vector<std::wstring>& args)
             throw hresult_invalid_argument(msg.c_str());
         }
     }
-
+    if (!m_inputImageFolderPath.empty())
+    {
+        PopulateInputImagePaths();
+    }
     SetupOutputDirectories(sBaseOutputPath, sPerfOutputPath, sPerIterationDataPath);
 
     CheckForInvalidArguments();
 }
 
+void CommandLineArgs::PopulateInputImagePaths()
+{
+    for (auto& it : std::filesystem::directory_iterator(m_inputImageFolderPath))
+    {
+        std::string path = it.path().string();
+        if (it.path().string().find(".png") != std::string::npos ||
+            it.path().string().find(".jpg") != std::string::npos ||
+            it.path().string().find(".jpeg") != std::string::npos)
+        {
+            std::wstring fileName;
+            fileName.assign(path.begin(), path.end());
+            m_imagePaths.push_back(fileName);
+        }
+    }
+}
+
 void CommandLineArgs::SetupOutputDirectories(const std::wstring& sBaseOutputPath,
                                              const std::wstring& sPerfOutputPath,
                                              const std::wstring& sPerIterationDataPath)
@@ -420,4 +447,8 @@ void CommandLineArgs::CheckForInvalidArguments()
     {
         throw hresult_invalid_argument(L"Cannot save tensor output if no input data is provided!");
     }
+    if (m_imagePaths.size() > 1 && IsSaveTensor())
+    {
+        throw hresult_not_implemented(L"Saving tensor output for multiple images isn't implemented.");
+    }
 }

Tools/WinMLRunner/src/CommandLineArgs.h

@@ -24,7 +24,7 @@ class CommandLineArgs
 
     BitmapInterpolationMode AutoScaleInterpMode() const { return m_autoScaleInterpMode; }
 
-    const std::wstring& ImagePath() const { return m_imagePath; }
+    const std::vector<std::wstring>& ImagePaths() const { return m_imagePaths; }
     const std::wstring& CsvPath() const { return m_csvData; }
     const std::wstring& OutputPath() const { return m_perfOutputPath; }
     const std::wstring& FolderPath() const { return m_modelFolderPath; }
@@ -38,7 +38,7 @@ class CommandLineArgs
     bool UseRGB() const
     {
         // If an image is specified without flags, we load it as a BGR image by default
-        return m_useRGB || (!m_imagePath.empty() && !m_useBGR && !m_useTensor);
+        return m_useRGB || (!m_imagePaths.empty() && !m_useBGR && !m_useTensor);
     }
 
     bool UseTensor() const
@@ -70,10 +70,10 @@ class CommandLineArgs
     bool IsGarbageInput() const
     {
         // When there is no image or csv input provided, then garbage input binding is used.
-        return m_imagePath.empty() && m_csvData.empty();
+        return m_imagePaths.empty() && m_csvData.empty();
     }
-    bool IsCSVInput() const { return m_imagePath.empty() && !m_csvData.empty(); }
-    bool IsImageInput() const { return !m_imagePath.empty() && m_csvData.empty(); }
+    bool IsCSVInput() const { return m_imagePaths.empty() && !m_csvData.empty(); }
+    bool IsImageInput() const { return !m_imagePaths.empty() && m_csvData.empty(); }
 
     uint32_t NumIterations() const { return m_numIterations; }
     uint32_t NumThreads() const { return m_numThreads; }
@@ -143,7 +143,8 @@ class CommandLineArgs
 
     std::wstring m_modelFolderPath;
     std::wstring m_modelPath;
-    std::wstring m_imagePath;
+    std::vector<std::wstring> m_imagePaths;
+    std::wstring m_inputImageFolderPath;
     std::wstring m_csvData;
     std::wstring m_inputData;
 #ifdef DXCORE_SUPPORTED_BUILD
@@ -161,4 +162,5 @@ class CommandLineArgs
     void CheckForInvalidArguments();
     void SetupOutputDirectories(const std::wstring& sBaseOutputPath, const std::wstring& sPerfOutputPath,
                                 const std::wstring& sPerIterationDataPath);
+    void PopulateInputImagePaths();
 };

Tools/WinMLRunner/src/Filehelper.cpp

@@ -22,4 +22,15 @@ namespace FileHelper
 
         return val;
     }
+    std::wstring GetAbsolutePath(std::wstring relativePath)
+    {
+        TCHAR** lppPart = { NULL };
+        wchar_t absolutePath[MAX_PATH] = { 0 };
+        errno_t err = GetFullPathName(relativePath.c_str(), MAX_PATH, absolutePath, lppPart);
+        if (err == 0)
+        {
+            throw HRESULT_FROM_WIN32(GetLastError());
+        }
+        return absolutePath;
+    }
 } // namespace FileHelper

Tools/WinMLRunner/src/Filehelper.h

@@ -4,4 +4,5 @@
 namespace FileHelper
 {
     std::wstring GetModulePath();
+    std::wstring GetAbsolutePath(std::wstring relativePath);
 }

Tools/WinMLRunner/src/OutputHelper.h

@@ -578,7 +578,7 @@ class OutputHelper
 
     void SetCSVFileName(const std::wstring& fileName) { m_csvFileName = fileName; }
 
-    void WritePerIterationPerformance(const CommandLineArgs& args, std::wstring model)
+    void WritePerIterationPerformance(const CommandLineArgs& args, const std::wstring model, const std::wstring imagePath)
     {
         if (m_csvFileNamePerIterationSummary.length() > 0)
         {
@@ -599,7 +599,7 @@ class OutputHelper
             std::string modelName = converter.to_bytes(model);
             std::string fileNameResultDevice = converter.to_bytes(m_fileNameResultDevice);
             std::string inputName = args.IsCSVInput() ? converter.to_bytes(args.CsvPath())
-                                                      : args.IsImageInput() ? converter.to_bytes(args.ImagePath()) : "";
+                                                      : args.IsImageInput() ? converter.to_bytes(imagePath) : "";
 
             if (bNewFile)
             {