Skip to content

Commit a8727a3

Browse files
pmbrown1055ryanlai2
pmbrown1055
authored and
ryanlai2
committed
Add -BaseOutputDir and more robust path support in general. Fix PerIterationPerf issue with Unit Test. (#216)
1 parent 850f924 commit a8727a3

File tree

7 files changed

+179
-116
lines changed

7 files changed

+179
-116
lines changed

‎Testing/WinMLRunnerTest/WinMLRunnerTest.cpp‎

Lines changed: 33 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ namespace WinMLRunnerTest
3838
static const std::wstring EXE_PATH = CURRENT_PATH + L"WinMLRunner.exe";
3939
static const std::wstring INPUT_FOLDER_PATH = CURRENT_PATH + L"test_folder_input";
4040
static const std::wstring OUTPUT_PATH = CURRENT_PATH + L"test_output.csv";
41-
static const std::wstring TENSOR_DATA_PATH = CURRENT_PATH + L"\\TestResult";
41+
static const std::wstring TENSOR_DATA_PATH = CURRENT_PATH + L"TestResult";
4242

4343
static std::wstring BuildCommand(std::initializer_list<std::wstring>&& arguments)
4444
{
@@ -52,13 +52,18 @@ namespace WinMLRunnerTest
5252
return commandLine;
5353
}
5454

55-
static size_t GetOutputCSVLineCount()
55+
static size_t GetOutputCSVLineCount(const std::wstring& path)
5656
{
5757
std::ifstream fin;
58-
fin.open(OUTPUT_PATH);
58+
fin.open(path);
5959
return static_cast<size_t>(std::count(std::istreambuf_iterator<char>(fin), std::istreambuf_iterator<char>(), '\n'));
6060
}
6161

62+
static size_t GetOutputCSVLineCount()
63+
{
64+
return GetOutputCSVLineCount(OUTPUT_PATH);
65+
}
66+
6267
static void RemoveModelsFromFolder(std::initializer_list<std::string>&& modelList)
6368
{
6469
//make test_models folder
@@ -262,7 +267,6 @@ public: TEST_CLASS_INITIALIZE(SetupClass) {
262267
// We need to expect one more line because of the header
263268
Assert::AreEqual(static_cast<size_t>(3), GetOutputCSVLineCount());
264269
}
265-
266270
TEST_METHOD(GarbageInputOnlyCpu)
267271
{
268272
const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx";
@@ -273,7 +277,6 @@ public: TEST_CLASS_INITIALIZE(SetupClass) {
273277
// We need to expect one more line because of the header
274278
Assert::AreEqual(static_cast<size_t>(2), GetOutputCSVLineCount());
275279
}
276-
277280
TEST_METHOD(GarbageInputOnlyGpu)
278281
{
279282
const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx";
@@ -554,7 +557,7 @@ public: TEST_CLASS_INITIALIZE(SetupClass) {
554557
const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx";
555558
const std::wstring inputPath = CURRENT_PATH + L"fish.png";
556559
const std::wstring command = BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath,
557-
L"-SaveTensorData", L"First", TENSOR_DATA_PATH, L"-GPU" });
560+
L"-SaveTensorData", L"First", L"-PerIterationPath", TENSOR_DATA_PATH, L"-GPU" });
558561
Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str()));
559562
Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Squeezenet_fish_input_GPU.csv",
560563
TENSOR_DATA_PATH + L"\\softmaxout_1GpuIteration1.csv"));
@@ -564,7 +567,7 @@ public: TEST_CLASS_INITIALIZE(SetupClass) {
564567
const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx";
565568
const std::wstring inputPath = CURRENT_PATH + L"fish.png";
566569
const std::wstring command = BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath,
567-
L"-SaveTensorData", L"First", TENSOR_DATA_PATH, L"-CPU" });
570+
L"-SaveTensorData", L"First", L"-PerIterationPath", TENSOR_DATA_PATH, L"-CPU" });
568571
Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str()));
569572
Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Squeezenet_fish_input_CPU.csv",
570573
TENSOR_DATA_PATH + L"\\softmaxout_1CpuIteration1.csv"));
@@ -575,7 +578,7 @@ public: TEST_CLASS_INITIALIZE(SetupClass) {
575578
const std::wstring modelPath = CURRENT_PATH + L"mnist.onnx";
576579
const std::wstring inputPath = CURRENT_PATH + L"mnist_28.png";
577580
const std::wstring command = BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath,
578-
L"-SaveTensorData", L"First", TENSOR_DATA_PATH, L"-GPU" });
581+
L"-SaveTensorData", L"First", L"-PerIterationPath", TENSOR_DATA_PATH, L"-GPU" });
579582
Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str()));
580583
Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Mnist_8_input_GPU.csv",
581584
TENSOR_DATA_PATH + L"\\Plus214_Output_0GpuIteration1.csv"));
@@ -585,7 +588,7 @@ public: TEST_CLASS_INITIALIZE(SetupClass) {
585588
const std::wstring modelPath = CURRENT_PATH + L"mnist.onnx";
586589
const std::wstring inputPath = CURRENT_PATH + L"mnist_28.png";
587590
const std::wstring command = BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath,
588-
L"-SaveTensorData", L"First", TENSOR_DATA_PATH, L"-CPU" });
591+
L"-SaveTensorData", L"First", L"-PerIterationPath", TENSOR_DATA_PATH, L"-CPU" });
589592
Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str()));
590593
Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Mnist_8_input_CPU.csv",
591594
TENSOR_DATA_PATH + L"\\Plus214_Output_0CpuIteration1.csv"));
@@ -595,7 +598,7 @@ public: TEST_CLASS_INITIALIZE(SetupClass) {
595598
const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet_fp16.onnx";
596599
const std::wstring inputPath = CURRENT_PATH + L"fish.png";
597600
const std::wstring command = BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath,
598-
L"-SaveTensorData", L"First", TENSOR_DATA_PATH, L"-GPU" });
601+
L"-SaveTensorData", L"First", L"-PerIterationPath", TENSOR_DATA_PATH, L"-GPU" });
599602
Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str()));
600603
Assert::AreEqual(true, CompareTensorsFP16(L"OutputTensorData\\Squeezenet_fp16_fish_input_GPU.csv",
601604
TENSOR_DATA_PATH + L"\\softmaxout_1GpuIteration1.csv"));
@@ -605,11 +608,23 @@ public: TEST_CLASS_INITIALIZE(SetupClass) {
605608
const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet_fp16.onnx";
606609
const std::wstring inputPath = CURRENT_PATH + L"fish.png";
607610
const std::wstring command = BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath,
608-
L"-SaveTensorData", L"First", TENSOR_DATA_PATH, L"-CPU" });
611+
L"-SaveTensorData", L"First", L"-PerIterationPath", TENSOR_DATA_PATH, L"-CPU" });
609612
Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str()));
610613
Assert::AreEqual(true, CompareTensorsFP16(L"OutputTensorData\\Squeezenet_fp16_fish_input_CPU.csv",
611614
TENSOR_DATA_PATH + L"\\softmaxout_1CpuIteration1.csv"));
612615
}
616+
TEST_METHOD(ProvidedImageInputOnlyCpuPerIterationPerformance)
617+
{
618+
const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx";
619+
const std::wstring command =
620+
BuildCommand({ EXE_PATH, L"-model", modelPath, L"-PerfOutput", OUTPUT_PATH, L"-perf",
621+
L"-SavePerIterationPerf", L"-BaseOutputPath", TENSOR_DATA_PATH,
622+
L"-PerIterationPath PerIterationData", L"-CPU" });
623+
Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str()));
624+
625+
// We need to expect one more line because of the header
626+
Assert::AreEqual(static_cast<size_t>(2), GetOutputCSVLineCount(TENSOR_DATA_PATH + L"\\PerIterationData\\Summary.csv"));
627+
}
613628
};
614629

615630
TEST_CLASS(CsvInputTest)
@@ -645,7 +660,7 @@ public: TEST_CLASS_INITIALIZE(SetupClass) {
645660
const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx";
646661
const std::wstring inputPath = CURRENT_PATH + L"fish.csv";
647662
const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-input", inputPath,
648-
L"-SaveTensorData", L"First", TENSOR_DATA_PATH, L"-GPU" });
663+
L"-SaveTensorData", L"First", L"-PerIterationPath", TENSOR_DATA_PATH, L"-GPU" });
649664
Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str()));
650665
Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Squeezenet_fish_input_GPU.csv",
651666
TENSOR_DATA_PATH + L"\\softmaxout_1GpuIteration1.csv"));
@@ -655,7 +670,7 @@ public: TEST_CLASS_INITIALIZE(SetupClass) {
655670
const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx";
656671
const std::wstring inputPath = CURRENT_PATH + L"fish.csv";
657672
const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-input", inputPath,
658-
L"-SaveTensorData", L"First", TENSOR_DATA_PATH, L"-GPU", L"-GPUBoundInput" });
673+
L"-SaveTensorData", L"First", L"-PerIterationPath", TENSOR_DATA_PATH, L"-GPU", L"-GPUBoundInput" });
659674
Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str()));
660675
Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Squeezenet_fish_input_GPU.csv",
661676
TENSOR_DATA_PATH + L"\\softmaxout_1GpuIteration1.csv"));
@@ -665,7 +680,7 @@ public: TEST_CLASS_INITIALIZE(SetupClass) {
665680
const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx";
666681
const std::wstring inputPath = CURRENT_PATH + L"fish.csv";
667682
const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-input", inputPath,
668-
L"-SaveTensorData", L"First", TENSOR_DATA_PATH, L"-CPU" });
683+
L"-SaveTensorData", L"First", L"-PerIterationPath", TENSOR_DATA_PATH, L"-CPU" });
669684
Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str()));
670685
Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Squeezenet_fish_input_CPU.csv",
671686
TENSOR_DATA_PATH + L"\\softmaxout_1CpuIteration1.csv"));
@@ -675,7 +690,7 @@ public: TEST_CLASS_INITIALIZE(SetupClass) {
675690
const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet_fp16.onnx";
676691
const std::wstring inputPath = CURRENT_PATH + L"fish.csv";
677692
const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-input", inputPath,
678-
L"-SaveTensorData", L"First", TENSOR_DATA_PATH, L"-GPU" });
693+
L"-SaveTensorData", L"First", L"-PerIterationPath", TENSOR_DATA_PATH, L"-GPU" });
679694
Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str()));
680695
Assert::AreEqual(true, CompareTensorsFP16(L"OutputTensorData\\Squeezenet_fp16_fish_input_GPU.csv",
681696
TENSOR_DATA_PATH + L"\\softmaxout_1GpuIteration1.csv"));
@@ -685,7 +700,7 @@ public: TEST_CLASS_INITIALIZE(SetupClass) {
685700
const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet_fp16.onnx";
686701
const std::wstring inputPath = CURRENT_PATH + L"fish.csv";
687702
const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-input", inputPath,
688-
L"-SaveTensorData", L"First", TENSOR_DATA_PATH, L"-CPU" });
703+
L"-SaveTensorData", L"First", L"-PerIterationPath", TENSOR_DATA_PATH, L"-CPU" });
689704
Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str()));
690705
Assert::AreEqual(true, CompareTensorsFP16(L"OutputTensorData\\Squeezenet_fp16_fish_input_CPU.csv",
691706
TENSOR_DATA_PATH + L"\\softmaxout_1CpuIteration1.csv"));
@@ -696,7 +711,7 @@ public: TEST_CLASS_INITIALIZE(SetupClass) {
696711
const std::wstring modelPath = CURRENT_PATH + L"mnist.onnx";
697712
const std::wstring inputPath = CURRENT_PATH + L"mnist_28.csv";
698713
const std::wstring command = BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath,
699-
L"-SaveTensorData", L"First", TENSOR_DATA_PATH, L"-GPU" });
714+
L"-SaveTensorData", L"First", L"-PerIterationPath", TENSOR_DATA_PATH, L"-GPU" });
700715
Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str()));
701716
Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Mnist_8_input_GPU.csv",
702717
TENSOR_DATA_PATH + L"\\Plus214_Output_0GpuIteration1.csv"));
@@ -706,7 +721,7 @@ public: TEST_CLASS_INITIALIZE(SetupClass) {
706721
const std::wstring modelPath = CURRENT_PATH + L"mnist.onnx";
707722
const std::wstring inputPath = CURRENT_PATH + L"mnist_28.csv";
708723
const std::wstring command = BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath,
709-
L"-SaveTensorData", L"First", TENSOR_DATA_PATH, L"-CPU" });
724+
L"-SaveTensorData", L"First", L"-PerIterationPath", TENSOR_DATA_PATH, L"-CPU" });
710725
Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str()));
711726
Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Mnist_8_input_CPU.csv",
712727
TENSOR_DATA_PATH + L"\\Plus214_Output_0CpuIteration1.csv"));

‎Tools/WinMLRunner/README.md‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,10 +47,11 @@ Required command-Line arguments:
4747
-Iterations : # times perf measurements will be run/averaged. (maximum: 1024 times)
4848
-Input <fully qualified path>: binds image or CSV to model
4949
-TopK <number>: print top <number> values in the result. Default to 1
50-
-PerfOutput [<fully qualified path>]: csv file to write the perf results to
50+
-BaseOutputPath [<fully qualified path>] : base output directory path for results, default to cwd
51+
-PerfOutput [<path>] : fully qualified or relative path including csv filename for perf results
5152
-SavePerIterationPerf : save per iteration performance results to csv file
52-
-SaveTensorData <saveMode folderPath>: saveMode: save first iteration or all iteration output tensor results to csv file [First, All]
53-
folderPath: Optional folder path can be specified to hold tensor data. It will be created if folder doesn't exist.
53+
-PerIterationPath <directory_path> : Relative or fully qualified path for per iteration and save tensor output results. If not specified a default(timestamped) folder will be created.
54+
-SaveTensorData <saveMode>: saveMode: save first iteration or all iteration output tensor results to csv file [First, All]
5455
-DebugEvaluate: Print evaluation debug output to debug console if debugger is present.
5556
-Terse: Terse Mode (suppresses repetitive console output)
5657
-AutoScale <interpolationMode>: Enable image autoscaling and set the interpolation mode [Nearest, Linear, Cubic, Fant]

‎Tools/WinMLRunner/src/BindingUtilities.h‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -629,8 +629,8 @@ namespace BindingUtilities
629629
{
630630
if (desc.Kind() == LearningModelFeatureKind::Tensor)
631631
{
632-
std::string name = to_string(desc.Name());
633-
if (args.IsSaveTensor() && args.SaveTensorMode() == "First" && iterationNum > 0)
632+
std::wstring name = desc.Name().c_str();
633+
if (args.IsSaveTensor() && args.SaveTensorMode() == L"First" && iterationNum > 0)
634634
{
635635
return;
636636
}
@@ -708,13 +708,13 @@ namespace BindingUtilities
708708
}
709709
if (!args.IsGarbageInput() && iterationNum == 0)
710710
{
711-
std::cout << "Outputting top " << args.TopK() << " values" << std::endl;
712-
std::cout << "Feature Name: " << name << std::endl;
711+
std::wcout << L"Outputting top " << args.TopK() << L" values" << std::endl;
712+
std::wcout << L"Feature Name: " << name << std::endl;
713713
for (auto& pair : maxKValues)
714714
{
715715
auto maxValue = pair.first;
716716
auto maxIndex = pair.second;
717-
std::wcout << " index: " << maxIndex << ", value: " << maxValue << std::endl;
717+
std::wcout << L" index: " << maxIndex << L", value: " << maxValue << std::endl;
718718
}
719719
}
720720
}

0 commit comments

Comments
 (0)