Skip to content

Commit d2b1522

Browse files
laks0209Ryan Lai
laks0209
authored and
Ryan Lai
committed
Changed formatting for Stringify types to be more consistent (#136) (#113)
1 parent ac6152d commit d2b1522

File tree

11 files changed

+4439
-72
lines changed

11 files changed

+4439
-72
lines changed

‎SharedContent/media/softmaxout1CpuIteration1.csv‎

Lines changed: 1001 additions & 0 deletions
Large diffs are not rendered by default.

‎SharedContent/media/softmaxout1GpuIteration1.csv‎

Lines changed: 1001 additions & 0 deletions
Large diffs are not rendered by default.

‎SharedContent/media/softmaxout1GpuIteration1Csv.csv‎

Lines changed: 1001 additions & 0 deletions
Large diffs are not rendered by default.

‎SharedContent/media/softmaxout1GpuIteration1Garbage.csv‎

Lines changed: 1001 additions & 0 deletions
Large diffs are not rendered by default.

‎Testing/WinMLRunnerTest/WinMLRunnerTest.cpp‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@
77
#include <fstream>
88
#include <algorithm>
99
#include <vector>
10+
#include <direct.h>
11+
#include <iomanip>
12+
#include <codecvt>
1013

1114
using namespace Microsoft::VisualStudio::CppUnitTestFramework;
1215
static HRESULT RunProc(LPWSTR commandLine)
@@ -68,6 +71,47 @@ namespace WinMLRunnerTest
6871
}
6972
}
7073

74+
bool CompareTensors(std::wstring trueTensor)
75+
{
76+
auto time = std::time(nullptr);
77+
struct tm localTime;
78+
localtime_s(&localTime, &time);
79+
std::ostringstream oss;
80+
oss << std::put_time(&localTime, "%Y-%m-%d_%H.%M.%S");
81+
82+
std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;
83+
std::string folderName = "PerIterationRun[" + oss.str() + "]\\softmaxout1GpuIteration1.csv";
84+
std::wstring csvTensor = converter.from_bytes(folderName);
85+
bool check = true;
86+
87+
if (trueTensor.length() > 0 && csvTensor.length() > 0)
88+
{
89+
std::ifstream trueFin;
90+
std::ifstream fin;
91+
trueFin.open(trueTensor, std::ios_base::app);
92+
fin.open(csvTensor);
93+
94+
std::string value;
95+
std::string trueValue;
96+
int i = 0;
97+
while (trueFin.good() && fin.good())
98+
{
99+
std::getline(trueFin, trueValue, ',');
100+
std::getline(trueFin, trueValue, '\n');
101+
std::getline(fin, value, ',');
102+
std::getline(fin, value, '\n');
103+
i++;
104+
if (i == 1 || i == 1002) continue;
105+
if (abs(std::stof(trueValue) - std::stof(value)) > 0.01)
106+
{
107+
check = false;
108+
}
109+
if (!check) break;
110+
}
111+
}
112+
return check;
113+
}
114+
71115
TEST_CLASS(GarbageInputTest)
72116
{
73117
public:
@@ -432,6 +476,13 @@ namespace WinMLRunnerTest
432476
// We need to expect one more line because of the header
433477
Assert::AreEqual(static_cast<size_t>(49), GetOutputCSVLineCount());
434478
}
479+
TEST_METHOD(GarbageInputOnlyGpuSaveTensor)
480+
{
481+
const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx";
482+
const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-output", OUTPUT_PATH,L"-SaveTensorData", L"First", L"-GPU", L"-RGB" });
483+
Assert::AreEqual(S_OK, RunProc((wchar_t *)command.c_str()));
484+
Assert::AreEqual(true, CompareTensors(L"softmaxout1GpuIteration1Garbage.csv"));
485+
}
435486
};
436487

437488
TEST_CLASS(ImageInputTest)
@@ -468,6 +519,22 @@ namespace WinMLRunnerTest
468519
const std::wstring command = BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath, L"-autoScale", L"Cubic" });
469520
Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str()));
470521
}
522+
TEST_METHOD(ProvidedImageInputOnlyGpuSaveTensor)
523+
{
524+
const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx";
525+
const std::wstring inputPath = CURRENT_PATH + L"fish.png";
526+
const std::wstring command = BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath, L"-SaveTensorData", L"First", L"-GPU" });
527+
Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str()));
528+
Assert::AreEqual(true, CompareTensors(L"softmaxout1GpuIteration1.csv"));
529+
}
530+
TEST_METHOD(ProvidedImageInputOnlyCpuSaveTensor)
531+
{
532+
const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx";
533+
const std::wstring inputPath = CURRENT_PATH + L"fish.png";
534+
const std::wstring command = BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath, L"-SaveTensorData", L"First", L"-CPU" });
535+
Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str()));
536+
Assert::AreEqual(true, CompareTensors(L"softmaxout1CpuIteration1.csv"));
537+
}
471538
};
472539

473540
TEST_CLASS(CsvInputTest)
@@ -488,6 +555,14 @@ namespace WinMLRunnerTest
488555
const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-input", inputPath });
489556
Assert::AreEqual(HRESULT_FROM_WIN32(ERROR_INVALID_PARAMETER), RunProc((wchar_t *)command.c_str()));
490557
}
558+
TEST_METHOD(ProvidedCSVInputSaveTensor)
559+
{
560+
const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx";
561+
const std::wstring inputPath = CURRENT_PATH + L"kitten_224.csv";
562+
const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-input", inputPath, L"-SaveTensorData", L"First", L"-GPU" });
563+
Assert::AreEqual(S_OK, RunProc((wchar_t *)command.c_str()));
564+
Assert::AreEqual(true, CompareTensors(L"softmaxout1GpuIteration1Csv.csv"));
565+
}
491566
};
492567

493568
TEST_CLASS(OtherTests)

‎Testing/WinMLRunnerTest/WinMLRunnerTest.vcxproj‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,52 @@
265265
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</DeploymentContent>
266266
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
267267
</Content>
268+
<Content Include="..\..\SharedContent\media\softmaxout1CpuIteration1.csv">
269+
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">false</ExcludedFromBuild>
270+
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</DeploymentContent>
271+
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</ExcludedFromBuild>
272+
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</DeploymentContent>
273+
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">false</ExcludedFromBuild>
274+
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</DeploymentContent>
275+
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">false</ExcludedFromBuild>
276+
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</DeploymentContent>
277+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
278+
</Content>
279+
<Content Include="..\..\SharedContent\media\softmaxout1GpuIteration1.csv">
280+
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">false</ExcludedFromBuild>
281+
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</DeploymentContent>
282+
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</ExcludedFromBuild>
283+
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</DeploymentContent>
284+
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">false</ExcludedFromBuild>
285+
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</DeploymentContent>
286+
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">false</ExcludedFromBuild>
287+
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</DeploymentContent>
288+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
289+
</Content>
290+
<Content Include="..\..\SharedContent\media\softmaxout1GpuIteration1Csv.csv">
291+
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">false</ExcludedFromBuild>
292+
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</DeploymentContent>
293+
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</ExcludedFromBuild>
294+
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</DeploymentContent>
295+
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">false</ExcludedFromBuild>
296+
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</DeploymentContent>
297+
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">false</ExcludedFromBuild>
298+
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</DeploymentContent>
299+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
300+
</Content>
301+
<Content Include="..\..\SharedContent\media\softmaxout1GpuIteration1Garbage.csv">
302+
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">false</ExcludedFromBuild>
303+
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</DeploymentContent>
304+
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</ExcludedFromBuild>
305+
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</DeploymentContent>
306+
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">false</ExcludedFromBuild>
307+
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</DeploymentContent>
308+
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">false</ExcludedFromBuild>
309+
<DeploymentContent Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</DeploymentContent>
310+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
311+
</Content>
312+
313+
268314
</ItemGroup>
269315
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
270316
<ImportGroup Label="ExtensionTargets">

‎Tools/WinMLRunner/src/BindingUtilities.h‎

Lines changed: 88 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
#include <time.h>
44
#include "Common.h"
55
#include "ModelBinding.h"
6-
#include "CommandLineArgs.h"
6+
#include "Windows.AI.Machinelearning.Native.h"
77

88
using namespace winrt::Windows::Media;
99
using namespace winrt::Windows::Storage;
@@ -13,6 +13,27 @@ using namespace winrt::Windows::Graphics::DirectX;
1313
using namespace winrt::Windows::Graphics::Imaging;
1414
using namespace winrt::Windows::Graphics::DirectX::Direct3D11;
1515

16+
inline size_t hash_data(void const* ptr, size_t const bytes) noexcept
17+
{
18+
#ifdef _WIN64
19+
constexpr size_t fnv_offset_basis = 14695981039346656037ULL;
20+
constexpr size_t fnv_prime = 1099511628211ULL;
21+
#else
22+
constexpr size_t fnv_offset_basis = 2166136261U;
23+
constexpr size_t fnv_prime = 16777619U;
24+
#endif
25+
size_t result = fnv_offset_basis;
26+
uint8_t const* const buffer = static_cast<uint8_t const*>(ptr);
27+
28+
for (size_t next = 0; next < bytes; ++next)
29+
{
30+
result ^= buffer[next];
31+
result *= fnv_prime;
32+
}
33+
34+
return result;
35+
}
36+
1637
namespace BindingUtilities
1738
{
1839
static unsigned int seed = 0;
@@ -494,4 +515,70 @@ namespace BindingUtilities
494515
std::cout << std::endl;
495516
}
496517
}
518+
519+
void SaveEvaluationResults(const LearningModel &model, const CommandLineArgs &args, const IMapView<hstring, winrt::Windows::Foundation::IInspectable>& results, OutputHelper &output, uint32_t iterationNum)
520+
{
521+
std::string iterationResult;
522+
size_t hash = 0;
523+
524+
for (auto&& desc : model.OutputFeatures())
525+
{
526+
if (desc.Kind() == LearningModelFeatureKind::Tensor)
527+
{
528+
std::wstring name(desc.Name());
529+
std::string tempName;
530+
for (int i = 0; i < name.size(); i++)
531+
{
532+
if (isalnum(name[i]))
533+
{
534+
tempName += name[i];
535+
}
536+
}
537+
TensorFeatureDescriptor tensorDescriptor = desc.as<TensorFeatureDescriptor>();
538+
TensorKind tensorKind = tensorDescriptor.TensorKind();
539+
540+
switch (tensorKind)
541+
{
542+
543+
case TensorKind::Float:
544+
{
545+
com_ptr<ITensorNative> itn = results.Lookup(desc.Name()).as<ITensorNative>();
546+
float* Tensor;
547+
uint32_t uCapacity;
548+
HRESULT(itn->GetBuffer(reinterpret_cast<BYTE**>(&Tensor), &uCapacity));
549+
hash = hash_data(Tensor, uCapacity);
550+
551+
UINT maxIndex = 0;
552+
auto maxValue = *Tensor;
553+
UINT size = uCapacity / sizeof(float_t);
554+
for (UINT i = 0; i < size; i++)
555+
{
556+
float val = *(Tensor+i);
557+
if (maxValue < val)
558+
{
559+
maxValue = val;
560+
maxIndex = i;
561+
}
562+
}
563+
iterationResult = "Index: " + std::to_string(maxIndex) + "; Value: " + std::to_string(maxValue);
564+
output.WriteTensorResultToCSV(Tensor, iterationNum, args, uCapacity, tempName);
565+
}
566+
break;
567+
568+
default:
569+
{
570+
std::cout << "BindingUtilities: output type not implemented.";
571+
}
572+
break;
573+
}
574+
output.SaveResult(iterationNum - 1, iterationResult, hash);
575+
}
576+
577+
else if (desc.Kind() == LearningModelFeatureKind::Sequence)
578+
{
579+
std::cout << "**********WARNING**********\nSequence Type Output Encountered\n Output Tensor not saved";
580+
std::cout << "**********WARNING**********\nSequence Type Output Encountered\n Output Tensor not saved";
581+
}
582+
}
583+
}
497584
};

‎Tools/WinMLRunner/src/CommandLineArgs.cpp‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ void CommandLineArgs::PrintUsage() {
2828
std::cout << " -PerfOutput optional:<fully qualified path>: csv file to write the perf results to" << std::endl;
2929
std::cout << " -IgnoreFirstRun : ignore the first run in the perf results when calculating the average" << std::endl;
3030
std::cout << " -SavePerIterationPerf : save per iteration performance results to csv file" << std::endl;
31+
std::cout << " -SaveTensorData <saveMode>: save first iteration or all iteration output tensor results to csv file [First, All]" << std::endl;
3132
std::cout << " -Debug: print trace logs" << std::endl;
3233
std::cout << " -Terse: Terse Mode (suppresses repetitive console output)" << std::endl;
3334
std::cout << " -AutoScale <interpolationMode>: Enable image autoscaling and set the interpolation mode [Nearest, Linear, Cubic, Fant]" << std::endl;
@@ -151,6 +152,24 @@ CommandLineArgs::CommandLineArgs(const std::vector<std::wstring>& args)
151152
return;
152153
}
153154
}
155+
else if ((_wcsicmp(args[i].c_str(), L"-SaveTensorData") == 0) && (i + 1 < args.size()))
156+
{
157+
m_saveTensor = true;
158+
if (_wcsicmp(args[++i].c_str(), L"First") == 0)
159+
{
160+
m_saveTensorMode = "First";
161+
}
162+
else if (_wcsicmp(args[i].c_str(), L"All") == 0)
163+
{
164+
m_saveTensorMode = "All";
165+
}
166+
else
167+
{
168+
std::cout << "Unknown Mode!" << std::endl;
169+
PrintUsage();
170+
return;
171+
}
172+
}
154173
else if ((_wcsicmp(args[i].c_str(), L"/?") == 0))
155174
{
156175
PrintUsage();

‎Tools/WinMLRunner/src/CommandLineArgs.h‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ class CommandLineArgs
1919
bool IsCreateDeviceOnClient() const { return m_createDeviceOnClient; }
2020
bool IsAutoScale() const { return m_autoScale; }
2121
bool IsOutputPerf() const { return m_perfOutput; }
22-
22+
bool IsSaveTensor() const { return m_saveTensor; }
2323

2424
BitmapInterpolationMode AutoScaleInterpMode() const { return m_autoScaleInterpMode; }
2525

@@ -98,6 +98,12 @@ class CommandLineArgs
9898
m_perfOutput = true;
9999
}
100100
void SetRunIterations(const uint32_t iterations) { m_numIterations = iterations; }
101+
102+
std::string SaveTensorMode() const
103+
{
104+
return m_saveTensorMode;
105+
}
106+
101107
private:
102108
bool m_perfCapture = false;
103109
bool m_useCPU = false;
@@ -118,6 +124,8 @@ class CommandLineArgs
118124
bool m_autoScale = false;
119125
bool m_perfOutput = false;
120126
BitmapInterpolationMode m_autoScaleInterpMode = BitmapInterpolationMode::Cubic;
127+
bool m_saveTensor = false;
128+
std::string m_saveTensorMode = "First";
121129

122130
std::wstring m_modelFolderPath;
123131
std::wstring m_modelPath;

0 commit comments

Comments
 (0)