Skip to content

Commit 014a092

Browse files
yskim1501Ryan Lai
yskim1501
authored and
Ryan Lai
committed
Pass in second value for -SaveTensorData parameter (#172)
* pass in optional second value for parameter -SaveTensorData and fix test cases to compare csv values correctly * Fix build break and update comments
1 parent 34124f0 commit 014a092

File tree

7 files changed

+123
-96
lines changed

7 files changed

+123
-96
lines changed

‎Testing/WinMLRunnerTest/WinMLRunnerTest.cpp‎

Lines changed: 88 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
#include <processthreadsapi.h>
55
#include <winnt.h>
66
#include <Winbase.h>
7+
#include <filesystem>
78
#include <fstream>
89
#include <algorithm>
910
#include <vector>
@@ -35,6 +36,7 @@ namespace WinMLRunnerTest
3536
static const std::wstring EXE_PATH = CURRENT_PATH + L"WinMLRunner.exe";
3637
static const std::wstring INPUT_FOLDER_PATH = CURRENT_PATH + L"test_folder_input";
3738
static const std::wstring OUTPUT_PATH = CURRENT_PATH + L"test_output.csv";
39+
static const std::wstring TENSOR_DATA_PATH = CURRENT_PATH + L"\\TestResult";
3840

3941
static std::wstring BuildCommand(std::initializer_list<std::wstring>&& arguments)
4042
{
@@ -71,50 +73,48 @@ namespace WinMLRunnerTest
7173
}
7274
}
7375

74-
bool CompareTensors(std::wstring trueTensor)
75-
{
76-
auto time = std::time(nullptr);
77-
struct tm localTime;
78-
localtime_s(&localTime, &time);
79-
std::ostringstream oss;
80-
oss << std::put_time(&localTime, "%Y-%m-%d_%H.%M.%S");
81-
82-
std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
83-
std::string folderName = "PerIterationRun[" + oss.str() + "]\\softmaxout1GpuIteration1.csv";
84-
std::wstring csvTensor = converter.from_bytes(folderName);
85-
bool check = true;
86-
87-
if (trueTensor.length() > 0 && csvTensor.length() > 0)
88-
{
89-
std::ifstream trueFin;
90-
std::ifstream fin;
91-
trueFin.open(trueTensor, std::ios_base::app);
92-
fin.open(csvTensor);
93-
94-
std::string value;
95-
std::string trueValue;
96-
int i = 0;
97-
while (trueFin.good() && fin.good())
98-
{
99-
std::getline(trueFin, trueValue, ',');
100-
std::getline(trueFin, trueValue, '\n');
101-
std::getline(fin, value, ',');
102-
std::getline(fin, value, '\n');
103-
i++;
104-
if (i == 1 || i == 1002) continue;
105-
if (abs(std::stof(trueValue) - std::stof(value)) > 0.01)
106-
{
107-
check = false;
108-
}
109-
if (!check) break;
110-
}
111-
}
112-
return check;
113-
}
114-
115-
TEST_CLASS(GarbageInputTest)
116-
{
117-
public:
76+
bool CompareTensors(const std::wstring &trueTensor, const std::wstring &outputTensor)
77+
{
78+
std::wstring csvTensor = outputTensor;
79+
bool check = true;
80+
81+
if (trueTensor.length() > 0 && csvTensor.length() > 0)
82+
{
83+
std::ifstream trueFin;
84+
std::ifstream fin;
85+
trueFin.open(trueTensor, std::ios_base::app);
86+
fin.open(csvTensor);
87+
88+
std::string value;
89+
std::string trueValue;
90+
int i = 0;
91+
92+
if (trueFin.fail() || fin.fail())
93+
{
94+
return false;
95+
}
96+
97+
while (!(trueFin.eof() || fin.eof()))
98+
{
99+
std::getline(trueFin, trueValue, ',');
100+
std::getline(trueFin, trueValue, '\n');
101+
std::getline(fin, value, ',');
102+
std::getline(fin, value, '\n');
103+
i++;
104+
if (i == 1 || i == 1002) continue;
105+
if (abs(std::stof(trueValue) - std::stof(value)) > 0.01)
106+
{
107+
check = false;
108+
break;
109+
}
110+
}
111+
}
112+
return check;
113+
}
114+
115+
TEST_CLASS(GarbageInputTest)
116+
{
117+
public:
118118
TEST_CLASS_INITIALIZE(SetupClass)
119119
{
120120
// Make test_folder_input folder before starting the tests
@@ -144,18 +144,23 @@ namespace WinMLRunnerTest
144144
TEST_METHOD_CLEANUP(CleanupMethod)
145145
{
146146
// Remove output.csv after each test
147+
// TODO: this will not work if each test runs in parallel. Use different file path for each test, and clean up at class setup
147148
std::remove(std::string(OUTPUT_PATH.begin(), OUTPUT_PATH.end()).c_str());
149+
try {
150+
std::filesystem::remove_all(std::string(TENSOR_DATA_PATH.begin(), TENSOR_DATA_PATH.end()).c_str());
151+
}
152+
catch (const std::filesystem::filesystem_error &) {}
148153
}
149154

150-
TEST_METHOD(GarbageInputCpuAndGpu)
151-
{
155+
TEST_METHOD(GarbageInputCpuAndGpu)
156+
{
152157
const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx";
153158
const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-PerfOutput", OUTPUT_PATH, L"-perf" });
154159
Assert::AreEqual(S_OK, RunProc((wchar_t *)command.c_str()));
155160

156161
// We need to expect one more line because of the header
157162
Assert::AreEqual(static_cast<size_t>(3), GetOutputCSVLineCount());
158-
}
163+
}
159164

160165
TEST_METHOD(GarbageInputOnlyCpu)
161166
{
@@ -476,14 +481,14 @@ namespace WinMLRunnerTest
476481
// We need to expect one more line because of the header
477482
Assert::AreEqual(static_cast<size_t>(49), GetOutputCSVLineCount());
478483
}
479-
TEST_METHOD(GarbageInputOnlyGpuSaveTensor)
480-
{
481-
const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx";
482-
const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-output", OUTPUT_PATH,L"-SaveTensorData", L"First", L"-GPU", L"-RGB" });
483-
Assert::AreEqual(S_OK, RunProc((wchar_t *)command.c_str()));
484-
Assert::AreEqual(true, CompareTensors(L"softmaxout1GpuIteration1Garbage.csv"));
485-
}
486-
};
484+
TEST_METHOD(GarbageInputOnlyGpuSaveTensor)
485+
{
486+
const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx";
487+
const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-PerfOutput", OUTPUT_PATH, L"-SaveTensorData", L"First", TENSOR_DATA_PATH, L"-GPU", L"-RGB" });
488+
Assert::AreEqual(S_OK, RunProc((wchar_t *)command.c_str()));
489+
Assert::AreEqual(true, CompareTensors(L"softmaxout1GpuIteration1Garbage.csv", TENSOR_DATA_PATH + L"\\softmaxout1GpuIteration1.csv"));
490+
}
491+
};
487492

488493
TEST_CLASS(ImageInputTest)
489494
{
@@ -519,22 +524,25 @@ namespace WinMLRunnerTest
519524
const std::wstring command = BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath, L"-autoScale", L"Cubic" });
520525
Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str()));
521526
}
522-
TEST_METHOD(ProvidedImageInputOnlyGpuSaveTensor)
523-
{
524-
const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx";
525-
const std::wstring inputPath = CURRENT_PATH + L"fish.png";
526-
const std::wstring command = BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath, L"-SaveTensorData", L"First", L"-GPU" });
527-
Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str()));
528-
Assert::AreEqual(true, CompareTensors(L"softmaxout1GpuIteration1.csv"));
529-
}
530-
TEST_METHOD(ProvidedImageInputOnlyCpuSaveTensor)
531-
{
532-
const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx";
533-
const std::wstring inputPath = CURRENT_PATH + L"fish.png";
534-
const std::wstring command = BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath, L"-SaveTensorData", L"First", L"-CPU" });
535-
Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str()));
536-
Assert::AreEqual(true, CompareTensors(L"softmaxout1CpuIteration1.csv"));
537-
}
527+
TEST_METHOD(ProvidedImageInputOnlyGpuSaveTensor)
528+
{
529+
const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx";
530+
const std::wstring inputPath = CURRENT_PATH + L"fish.png";
531+
const std::wstring command = BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath,
532+
L"-SaveTensorData", L"First", TENSOR_DATA_PATH, L"-GPU" });
533+
Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str()));
534+
Assert::AreEqual(true, CompareTensors(L"softmaxout1GpuIteration1.csv", TENSOR_DATA_PATH + L"\\softmaxout1GpuIteration1.csv"));
535+
}
536+
TEST_METHOD(ProvidedImageInputOnlyCpuSaveTensor)
537+
{
538+
const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx";
539+
const std::wstring inputPath = CURRENT_PATH + L"fish.png";
540+
const std::wstring tensorDataPath = CURRENT_PATH + L"TestResult\\ProvidedImageInputOnlyCpuSaveTensor.csv";
541+
const std::wstring command = BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath,
542+
L"-SaveTensorData", L"First", TENSOR_DATA_PATH, L"-CPU" });
543+
Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str()));
544+
Assert::AreEqual(true, CompareTensors(L"softmaxout1CpuIteration1.csv", TENSOR_DATA_PATH + L"\\softmaxout1CpuIteration1.csv"));
545+
}
538546
};
539547

540548
TEST_CLASS(CsvInputTest)
@@ -555,14 +563,15 @@ namespace WinMLRunnerTest
555563
const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-input", inputPath });
556564
Assert::AreEqual(HRESULT_FROM_WIN32(ERROR_INVALID_PARAMETER), RunProc((wchar_t *)command.c_str()));
557565
}
558-
TEST_METHOD(ProvidedCSVInputSaveTensor)
559-
{
560-
const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx";
561-
const std::wstring inputPath = CURRENT_PATH + L"kitten_224.csv";
562-
const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-input", inputPath, L"-SaveTensorData", L"First", L"-GPU" });
563-
Assert::AreEqual(S_OK, RunProc((wchar_t *)command.c_str()));
564-
Assert::AreEqual(true, CompareTensors(L"softmaxout1GpuIteration1Csv.csv"));
565-
}
566+
TEST_METHOD(ProvidedCSVInputSaveTensor)
567+
{
568+
const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx";
569+
const std::wstring inputPath = CURRENT_PATH + L"fish.csv";
570+
const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-input", inputPath, L"-SaveTensorData",
571+
L"First", TENSOR_DATA_PATH, L"-GPU" });
572+
Assert::AreEqual(S_OK, RunProc((wchar_t *)command.c_str()));
573+
Assert::AreEqual(true, CompareTensors(L"softmaxout1GpuIteration1.csv", TENSOR_DATA_PATH + L"\\softmaxout1GpuIteration1.csv"));
574+
}
566575
};
567576

568577
TEST_CLASS(OtherTests)

‎Testing/WinMLRunnerTest/WinMLRunnerTest.vcxproj‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@
9797
<AdditionalIncludeDirectories>$(VCInstallDir)UnitTest\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
9898
<PreprocessorDefinitions>WIN32;_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
9999
<UseFullPaths>true</UseFullPaths>
100+
<LanguageStandard>stdcpp17</LanguageStandard>
100101
</ClCompile>
101102
<Link>
102103
<SubSystem>Windows</SubSystem>
@@ -112,6 +113,7 @@
112113
<AdditionalIncludeDirectories>$(VCInstallDir)UnitTest\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
113114
<PreprocessorDefinitions>_DEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
114115
<UseFullPaths>true</UseFullPaths>
116+
<LanguageStandard>stdcpp17</LanguageStandard>
115117
</ClCompile>
116118
<Link>
117119
<SubSystem>Windows</SubSystem>
@@ -130,6 +132,7 @@
130132
<PreprocessorDefinitions>WIN32;NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
131133
<UseFullPaths>true</UseFullPaths>
132134
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
135+
<LanguageStandard>stdcpp17</LanguageStandard>
133136
</ClCompile>
134137
<Link>
135138
<SubSystem>Windows</SubSystem>
@@ -150,6 +153,7 @@
150153
<PreprocessorDefinitions>NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
151154
<UseFullPaths>true</UseFullPaths>
152155
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
156+
<LanguageStandard>stdcpp17</LanguageStandard>
153157
</ClCompile>
154158
<Link>
155159
<SubSystem>Windows</SubSystem>
@@ -276,7 +280,7 @@
276280
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</DeploymentContent>
277281
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
278282
</Content>
279-
<Content Include="..\..\SharedContent\media\softmaxout1GpuIteration1.csv">
283+
<Content Include="..\..\SharedContent\media\softmaxout1GpuIteration1.csv">
280284
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">false</ExcludedFromBuild>
281285
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</DeploymentContent>
282286
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</ExcludedFromBuild>
@@ -287,7 +291,7 @@
287291
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</DeploymentContent>
288292
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
289293
</Content>
290-
<Content Include="..\..\SharedContent\media\softmaxout1GpuIteration1Csv.csv">
294+
<Content Include="..\..\SharedContent\media\softmaxout1GpuIteration1Csv.csv">
291295
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">false</ExcludedFromBuild>
292296
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</DeploymentContent>
293297
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</ExcludedFromBuild>
@@ -298,7 +302,7 @@
298302
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</DeploymentContent>
299303
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
300304
</Content>
301-
<Content Include="..\..\SharedContent\media\softmaxout1GpuIteration1Garbage.csv">
305+
<Content Include="..\..\SharedContent\media\softmaxout1GpuIteration1Garbage.csv">
302306
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">false</ExcludedFromBuild>
303307
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</DeploymentContent>
304308
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</ExcludedFromBuild>
@@ -309,8 +313,6 @@
309313
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</DeploymentContent>
310314
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
311315
</Content>
312-
313-
314316
</ItemGroup>
315317
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
316318
<ImportGroup Label="ExtensionTargets">

‎Tools/WinMLRunner/WinMLRunnerScenarios.vcxproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@
139139
<LanguageStandard>stdcpp17</LanguageStandard>
140140
<PrecompiledHeaderFile />
141141
<PrecompiledHeaderOutputFile />
142+
<RuntimeLibrary>MultiThreaded</RuntimeLibrary>
142143
</ClCompile>
143144
<Link>
144145
<SubSystem>Windows</SubSystem>

‎Tools/WinMLRunner/src/CommandLineArgs.cpp‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
#include <iostream>
44
#include "CommandLineArgs.h"
55
#include <apiquery2.h>
6+
#include <ctime>
7+
#include <iomanip>
68

79
using namespace Windows::AI::MachineLearning;
810

@@ -30,14 +32,15 @@ void CommandLineArgs::PrintUsage() {
3032
std::cout << " -Input <fully qualified path>: binds image or CSV to model" << std::endl;
3133
std::cout << " -PerfOutput optional:<fully qualified path>: csv file to write the perf results to" << std::endl;
3234
std::cout << " -SavePerIterationPerf : save per iteration performance results to csv file" << std::endl;
33-
std::cout << " -SaveTensorData <saveMode>: save first iteration or all iteration output tensor results to csv file [First, All]" << std::endl;
35+
std::cout << " -SaveTensorData <saveMode folderPath>: saveMode: save first iteration or all iteration output tensor results to csv file [First, All]" << std::endl;
36+
std::cout << " folderPath: Optional folder path can be specified to hold tensor data. It will be created if folder doesn't exist." << std::endl;
3437
std::cout << " -Debug: print trace logs" << std::endl;
3538
std::cout << " -Terse: Terse Mode (suppresses repetitive console output)" << std::endl;
3639
std::cout << " -AutoScale <interpolationMode>: Enable image autoscaling and set the interpolation mode [Nearest, Linear, Cubic, Fant]" << std::endl;
3740
std::cout << std::endl;
3841
std::cout << "Concurrency Options:" << std::endl;
3942
std::cout << " -ConcurrentLoad: load models concurrently" << std::endl;
40-
std::cout << " -NumThreads <number>: number of threads to load a model. By default this will be the number of model files to be executed." << std::endl;
43+
std::cout << " -NumThreads <number>: number of threads to load a model. By default this will be the number of model files to be executed" << std::endl;
4144
std::cout << " -ThreadInterval <milliseconds>: interval time between two thread creations in milliseconds" << std::endl;
4245
}
4346

@@ -191,6 +194,21 @@ CommandLineArgs::CommandLineArgs(const std::vector<std::wstring> &args)
191194
PrintUsage();
192195
throw hresult_invalid_argument(L"Unknown Mode!");
193196
}
197+
try
198+
{
199+
// This is to check for second optional value for -SaveTensorData to specify path
200+
CheckNextArgument(args, i);
201+
SetTensorOutputPath(args[++i].c_str());
202+
}
203+
catch (...) {
204+
// set to default path
205+
auto time = std::time(nullptr);
206+
struct tm localTime;
207+
localtime_s(&localTime, &time);
208+
std::wostringstream oss;
209+
oss << std::put_time(&localTime, L"%Y-%m-%d_%H.%M.%S");
210+
SetTensorOutputPath(L"\\PerIterationRun[" + oss.str() + L"]");
211+
}
194212
}
195213
else if (_wcsicmp(args[i].c_str(), L"-version") == 0)
196214
{

‎Tools/WinMLRunner/src/CommandLineArgs.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ class CommandLineArgs
2929
const std::wstring& OutputPath() const { return m_perfOutputPath; }
3030
const std::wstring& FolderPath() const { return m_modelFolderPath; }
3131
const std::wstring& ModelPath() const { return m_modelPath; }
32+
const std::wstring& TensorOutputPath() const { return m_tensorOutputPath; }
3233
UINT GetGPUAdapterIndex() const { return m_adapterIndex; }
3334

3435
bool UseRGB() const
@@ -96,6 +97,7 @@ class CommandLineArgs
9697

9798

9899
void SetModelPath(const std::wstring& modelPath) { m_modelPath = modelPath; }
100+
void SetTensorOutputPath(const std::wstring& tensorOutputPath) { m_tensorOutputPath = tensorOutputPath; }
99101
void SetInputDataPath(const std::wstring& inputDataPath) { m_inputData = inputDataPath; }
100102
void SetNumThreads(unsigned numThreads) { m_numThreads = numThreads; }
101103
void SetThreadInterval(unsigned threadInterval) { m_threadInterval = threadInterval; }
@@ -143,6 +145,7 @@ class CommandLineArgs
143145
std::wstring m_csvData;
144146
std::wstring m_inputData;
145147
std::wstring m_perfOutputPath;
148+
std::wstring m_tensorOutputPath;
146149
uint32_t m_numIterations = 1;
147150
uint32_t m_numThreads = 1;
148151
uint32_t m_threadInterval = 0;

‎Tools/WinMLRunner/src/OutputHelper.h‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -485,16 +485,10 @@ class OutputHelper
485485
m_outputTensorHash[iterationNum] = hashcode;
486486
}
487487

488-
void SetDefaultPerIterationFolder()
488+
void SetDefaultPerIterationFolder(const std::wstring &folderName)
489489
{
490-
auto time = std::time(nullptr);
491-
struct tm localTime;
492-
localtime_s(&localTime, &time);
493-
std::string cur_dir = _getcwd(NULL, 0);
494-
std::ostringstream oss;
495-
oss << std::put_time(&localTime, "%Y-%m-%d_%H.%M.%S");
496-
std::string folderName = "\\PerIterationRun[" + oss.str() + "]";
497-
m_folderNamePerIteration = cur_dir + folderName;
490+
std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
491+
m_folderNamePerIteration = converter.to_bytes(folderName);
498492
if (_mkdir(m_folderNamePerIteration.c_str()) != 0)
499493
std::cout << "Folder cannot be created";
500494
}

0 commit comments

Comments
 (0)