Change binding data to not memset 0, make fp16 tensor tests more lenient (#180)

* Memset garbage binding input to 2 instead of 0

* Make FP16 tests more lenient

* Refactored Tensor creation to be cleaner. Garbage data is now arbitrary

* Added check when creating commandline args so that user cannot save tensors if no input data is provided

Author: Ryan Lai

Testing/WinMLRunnerTest/WinMLRunnerTest.cpp

@@ -75,7 +75,11 @@ namespace WinMLRunnerTest
         }
     }
 
-    bool CompareTensors(const std::wstring &expectedOutputTensorFile, const std::wstring &actualOutputTensorFile)
+    bool CompareTensorsProvidedEpsilonAndRelativeTolerance(
+        const std::wstring &expectedOutputTensorFile,
+        const std::wstring &actualOutputTensorFile,
+        float relativeTolerance,
+        float epsilon)
     {
         bool check = false;
         std::ifstream expectedFileStream;
@@ -88,8 +92,6 @@ namespace WinMLRunnerTest
         {
             return false;
         }
-        // hard coded, might need to be modified later.
-        float relativeTolerance = 0.003f;
         bool isFirstRow = true;
         while (!(expectedFileStream.eof() || actualFileStream.eof()))
         {
@@ -104,14 +106,24 @@ namespace WinMLRunnerTest
             }
             float actualValueNum = (actualValue == "") ? 0 : std::stof(actualValue);
             float expectedValueNum = (expectedValue == "") ? 0 : std::stof(expectedValue);
-            if (std::abs(actualValueNum - expectedValueNum) > relativeTolerance * std::abs(expectedValueNum) + relativeTolerance) //Check if the values are too different.
+            if (std::abs(actualValueNum - expectedValueNum) > relativeTolerance * std::abs(expectedValueNum) + epsilon) //Check if the values are too different.
             {
                 return false;
             }
         }
         return true;
     }
 
+    bool CompareTensors(const std::wstring &expectedOutputTensorFile, const std::wstring &actualOutputTensorFile)
+    {
+        return CompareTensorsProvidedEpsilonAndRelativeTolerance(expectedOutputTensorFile, actualOutputTensorFile, 0.003f, 0);
+    }
+
+    bool CompareTensorsFP16(const std::wstring &expectedOutputTensorFile, const std::wstring &actualOutputTensorFile)
+    {
+        return CompareTensorsProvidedEpsilonAndRelativeTolerance(expectedOutputTensorFile, actualOutputTensorFile, 0.06f, 0);
+    }
+
     TEST_CLASS(GarbageInputTest)
     {
     public:
@@ -480,35 +492,6 @@ namespace WinMLRunnerTest
             // We need to expect one more line because of the header
             Assert::AreEqual(static_cast<size_t>(49), GetOutputCSVLineCount());
         }
-        TEST_METHOD(GarbageInputOnlyGpuSaveTensor)
-        {
-            const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx";
-            const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-PerfOutput", OUTPUT_PATH, L"-SaveTensorData", L"First", TENSOR_DATA_PATH, L"-GPU"});
-            Assert::AreEqual(S_OK, RunProc((wchar_t *)command.c_str()));
-            Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Squeezenet_garbage_input_GPU.csv", TENSOR_DATA_PATH + L"\\softmaxout_1GpuIteration1.csv"));
-        }
-        TEST_METHOD(GarbageInputOnlyCpuSaveTensor)
-        {
-            const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet.onnx";
-            const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-PerfOutput", OUTPUT_PATH, L"-SaveTensorData", L"First", TENSOR_DATA_PATH, L"-CPU" });
-            Assert::AreEqual(S_OK, RunProc((wchar_t *)command.c_str()));
-            Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Squeezenet_garbage_input_CPU.csv", TENSOR_DATA_PATH + L"\\softmaxout_1CpuIteration1.csv"));
-        }
-
-        TEST_METHOD(GarbageInputOnlyGpuSaveTensorFp16)
-        {
-            const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet_fp16.onnx";
-            const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-PerfOutput", OUTPUT_PATH, L"-SaveTensorData", L"First", TENSOR_DATA_PATH, L"-GPU" });
-            Assert::AreEqual(S_OK, RunProc((wchar_t *)command.c_str()));
-            Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Squeezenet_fp16_garbage_input_GPU.csv", TENSOR_DATA_PATH + L"\\softmaxout_1GpuIteration1.csv"));
-        }
-        TEST_METHOD(GarbageInputOnlyCpuSaveTensorFp16)
-        {
-            const std::wstring modelPath = CURRENT_PATH + L"SqueezeNet_fp16.onnx";
-            const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-PerfOutput", OUTPUT_PATH, L"-SaveTensorData", L"First", TENSOR_DATA_PATH, L"-CPU" });
-            Assert::AreEqual(S_OK, RunProc((wchar_t *)command.c_str()));
-            Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Squeezenet_fp16_garbage_input_CPU.csv", TENSOR_DATA_PATH + L"\\softmaxout_1CpuIteration1.csv"));
-        }
     };
 
     TEST_CLASS(ImageInputTest)
@@ -579,7 +562,7 @@ namespace WinMLRunnerTest
             const std::wstring command = BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath,
                 L"-SaveTensorData", L"First", TENSOR_DATA_PATH, L"-GPU" });
             Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str()));
-            Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Squeezenet_fp16_fish_input_GPU.csv", TENSOR_DATA_PATH + L"\\softmaxout_1GpuIteration1.csv"));
+            Assert::AreEqual(true, CompareTensorsFP16(L"OutputTensorData\\Squeezenet_fp16_fish_input_GPU.csv", TENSOR_DATA_PATH + L"\\softmaxout_1GpuIteration1.csv"));
         }
         TEST_METHOD(ProvidedImageInputOnlyCpuSaveTensorFp16)
         {
@@ -588,7 +571,7 @@ namespace WinMLRunnerTest
             const std::wstring command = BuildCommand({ EXE_PATH, L"-model ", modelPath, L"-input", inputPath,
                 L"-SaveTensorData", L"First", TENSOR_DATA_PATH, L"-CPU" });
             Assert::AreEqual(S_OK, RunProc((wchar_t*)command.c_str()));
-            Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Squeezenet_fp16_fish_input_CPU.csv", TENSOR_DATA_PATH + L"\\softmaxout_1CpuIteration1.csv"));
+            Assert::AreEqual(true, CompareTensorsFP16(L"OutputTensorData\\Squeezenet_fp16_fish_input_CPU.csv", TENSOR_DATA_PATH + L"\\softmaxout_1CpuIteration1.csv"));
         }
     };
 
@@ -645,7 +628,7 @@ namespace WinMLRunnerTest
             const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-input", inputPath, L"-SaveTensorData",
                 L"First", TENSOR_DATA_PATH, L"-GPU" });
             Assert::AreEqual(S_OK, RunProc((wchar_t *)command.c_str()));
-            Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Squeezenet_fp16_fish_input_GPU.csv", TENSOR_DATA_PATH + L"\\softmaxout_1GpuIteration1.csv"));
+            Assert::AreEqual(true, CompareTensorsFP16(L"OutputTensorData\\Squeezenet_fp16_fish_input_GPU.csv", TENSOR_DATA_PATH + L"\\softmaxout_1GpuIteration1.csv"));
         }
         TEST_METHOD(ProvidedCSVInputCPUSaveTensorFp16)
         {
@@ -654,7 +637,7 @@ namespace WinMLRunnerTest
             const std::wstring command = BuildCommand({ EXE_PATH, L"-model", modelPath, L"-input", inputPath, L"-SaveTensorData",
                 L"First", TENSOR_DATA_PATH, L"-CPU" });
             Assert::AreEqual(S_OK, RunProc((wchar_t *)command.c_str()));
-            Assert::AreEqual(true, CompareTensors(L"OutputTensorData\\Squeezenet_fp16_fish_input_CPU.csv", TENSOR_DATA_PATH + L"\\softmaxout_1CpuIteration1.csv"));
+            Assert::AreEqual(true, CompareTensorsFP16(L"OutputTensorData\\Squeezenet_fp16_fish_input_CPU.csv", TENSOR_DATA_PATH + L"\\softmaxout_1CpuIteration1.csv"));
         }
     };
 

Tools/WinMLRunner/src/BindingUtilities.h

@@ -13,6 +13,36 @@ using namespace winrt::Windows::Graphics::DirectX;
 using namespace winrt::Windows::Graphics::Imaging;
 using namespace winrt::Windows::Graphics::DirectX::Direct3D11;
 
+template <TensorKind T> struct TensorKindToType { static_assert(true, "No TensorKind mapped for given type!"); };
+template <> struct TensorKindToType<TensorKind::UInt8> { typedef uint8_t Type; };
+template <> struct TensorKindToType<TensorKind::Int8> { typedef uint8_t Type; };
+template <> struct TensorKindToType<TensorKind::UInt16> { typedef uint16_t Type; };
+template <> struct TensorKindToType<TensorKind::Int16> { typedef int16_t Type; };
+template <> struct TensorKindToType<TensorKind::UInt32> { typedef uint32_t Type; };
+template <> struct TensorKindToType<TensorKind::Int32> { typedef int32_t Type; };
+template <> struct TensorKindToType<TensorKind::UInt64> { typedef uint64_t Type; };
+template <> struct TensorKindToType<TensorKind::Int64> { typedef int64_t Type; };
+template <> struct TensorKindToType<TensorKind::Boolean> { typedef boolean Type; };
+template <> struct TensorKindToType<TensorKind::Double> { typedef double Type; };
+template <> struct TensorKindToType<TensorKind::Float> { typedef float Type; };
+template <> struct TensorKindToType<TensorKind::Float16> { typedef float Type; };
+template <> struct TensorKindToType<TensorKind::String> { typedef winrt::hstring Type; };
+
+template <TensorKind T> struct TensorKindToValue { static_assert(true, "No TensorKind mapped for given type!"); };
+template <> struct TensorKindToValue<TensorKind::UInt8> { typedef TensorUInt8Bit Type; };
+template <> struct TensorKindToValue<TensorKind::Int8> { typedef TensorInt8Bit Type; };
+template <> struct TensorKindToValue<TensorKind::UInt16> { typedef TensorUInt16Bit Type; };
+template <> struct TensorKindToValue<TensorKind::Int16> { typedef TensorInt16Bit Type; };
+template <> struct TensorKindToValue<TensorKind::UInt32> { typedef TensorUInt32Bit Type; };
+template <> struct TensorKindToValue<TensorKind::Int32> { typedef TensorInt32Bit Type; };
+template <> struct TensorKindToValue<TensorKind::UInt64> { typedef TensorUInt64Bit Type; };
+template <> struct TensorKindToValue<TensorKind::Int64> { typedef TensorInt64Bit Type; };
+template <> struct TensorKindToValue<TensorKind::Boolean> { typedef TensorBoolean Type; };
+template <> struct TensorKindToValue<TensorKind::Double> { typedef TensorDouble Type; };
+template <> struct TensorKindToValue<TensorKind::Float> { typedef TensorFloat Type; };
+template <> struct TensorKindToValue<TensorKind::Float16> { typedef TensorFloat16Bit Type; };
+template <> struct TensorKindToValue<TensorKind::String> { typedef TensorString Type; };
+
 namespace BindingUtilities
 {
     static unsigned int seed = 0;
@@ -175,6 +205,41 @@ namespace BindingUtilities
         return elementStrings;
     }
 
+    template <TensorKind T>
+    static ITensor CreateTensor(
+        const CommandLineArgs& args,
+        std::vector<std::string>& tensorStringInput,
+        TensorFeatureDescriptor& tensorDescriptor)
+    {
+        using TensorValue = typename TensorKindToValue<T>::Type;
+        using DataType = typename TensorKindToType<T>::Type;
+
+        if (!args.CsvPath().empty())
+        {
+            ModelBinding<DataType> binding(tensorDescriptor);
+            WriteDataToBinding<DataType>(tensorStringInput, binding);
+            return TensorValue::CreateFromArray(binding.GetShapeBuffer(), binding.GetDataBuffer());
+        }
+        else if(args.IsGarbageInput())
+        {
+            auto tensorValue = TensorValue::Create(tensorDescriptor.Shape());
+
+            com_ptr<ITensorNative> spTensorValueNative;
+            tensorValue.as(spTensorValueNative);
+
+            BYTE* actualData;
+            uint32_t actualSizeInBytes;
+            spTensorValueNative->GetBuffer(&actualData, &actualSizeInBytes);
+
+            return tensorValue;
+        }
+        else
+        {
+            //Creating Tensors for Input Images haven't been added yet.
+            throw hresult_not_implemented(L"Creating Tensors for Input Images haven't been implemented yet!");
+        }
+    }
+
     // Binds tensor floats, ints, doubles from CSV data.
     ITensor CreateBindableTensor(const ILearningModelFeatureDescriptor& description, const CommandLineArgs& args)
     {
@@ -188,6 +253,10 @@ namespace BindingUtilities
         }
 
         std::vector<std::string> elementStrings;
+        if (!args.CsvPath().empty())
+        {
+            elementStrings = ParseCSVElementStrings(args.CsvPath());
+        }
         switch (tensorDescriptor.TensorKind())
         {
             case TensorKind::Undefined:
@@ -197,167 +266,57 @@ namespace BindingUtilities
             }
             case TensorKind::Float:
             {
-                ModelBinding<float> binding(description);
-                if (args.IsGarbageInput())
-                {
-                    memset(binding.GetData(), 0, sizeof(float) * binding.GetDataBufferSize());
-                }
-                else
-                {
-                    elementStrings = ParseCSVElementStrings(args.CsvPath());
-                    WriteDataToBinding<float>(elementStrings, binding);
-                }
-                return TensorFloat::CreateFromArray(binding.GetShapeBuffer(), binding.GetDataBuffer());
+                return CreateTensor<TensorKind::Float>(args, elementStrings, tensorDescriptor);
             }
             break;
             case TensorKind::Float16:
             {
-                ModelBinding<float> binding(description);
-                if (args.IsGarbageInput())
-                {
-                    memset(binding.GetData(), 0, sizeof(float) * binding.GetDataBufferSize());
-                }
-                else
-                {
-                    elementStrings = ParseCSVElementStrings(args.CsvPath());
-                    WriteDataToBinding<float>(elementStrings, binding);
-                }
-                return TensorFloat16Bit::CreateFromArray(binding.GetShapeBuffer(), binding.GetDataBuffer());
+                return CreateTensor<TensorKind::Float16>(args, elementStrings, tensorDescriptor);
             }
             break;
             case TensorKind::Double:
             {
-                ModelBinding<double> binding(description);
-                if (args.IsGarbageInput())
-                {
-                    memset(binding.GetData(), 0, sizeof(double) * binding.GetDataBufferSize());
-                }
-                else
-                {
-                    elementStrings = ParseCSVElementStrings(args.CsvPath());
-                    WriteDataToBinding<double>(elementStrings, binding);
-                }
-                return TensorDouble::CreateFromArray(binding.GetShapeBuffer(), binding.GetDataBuffer());
+                return CreateTensor<TensorKind::Double>(args, elementStrings, tensorDescriptor);
             }
             break;
             case TensorKind::Int8:
             {
-                ModelBinding<uint8_t> binding(description);
-                if (args.IsGarbageInput())
-                {
-                    memset(binding.GetData(), 0, sizeof(uint8_t) * binding.GetDataBufferSize());
-                }
-                else
-                {
-                    elementStrings = ParseCSVElementStrings(args.CsvPath());
-                    WriteDataToBinding<uint8_t>(elementStrings, binding);
-                }
-                return TensorInt8Bit::CreateFromArray(binding.GetShapeBuffer(), binding.GetDataBuffer());
+                return CreateTensor<TensorKind::Int8>(args, elementStrings, tensorDescriptor);
             }
             break;
             case TensorKind::UInt8:
             {
-                ModelBinding<uint8_t> binding(description);
-                if (args.IsGarbageInput())
-                {
-                    memset(binding.GetData(), 0, sizeof(uint8_t) * binding.GetDataBufferSize());
-                }
-                else
-                {
-                    elementStrings = ParseCSVElementStrings(args.CsvPath());
-                    WriteDataToBinding<uint8_t>(elementStrings, binding);
-                }
-                return TensorUInt8Bit::CreateFromArray(binding.GetShapeBuffer(), binding.GetDataBuffer());
+                return CreateTensor<TensorKind::UInt8>(args, elementStrings, tensorDescriptor);
             }
             break;
             case TensorKind::Int16:
             {
-                ModelBinding<int16_t> binding(description);
-                if (args.IsGarbageInput())
-                {
-                    memset(binding.GetData(), 0, sizeof(int16_t) * binding.GetDataBufferSize());
-                }
-                else
-                {
-                    elementStrings = ParseCSVElementStrings(args.CsvPath());
-                    WriteDataToBinding<int16_t>(elementStrings, binding);
-                }
-                return TensorInt16Bit::CreateFromArray(binding.GetShapeBuffer(), binding.GetDataBuffer());
+                return CreateTensor<TensorKind::Int16>(args, elementStrings, tensorDescriptor);
             }
             break;
             case TensorKind::UInt16:
             {
-                ModelBinding<uint16_t> binding(description);
-                if (args.IsGarbageInput())
-                {
-                    memset(binding.GetData(), 0, sizeof(uint16_t) * binding.GetDataBufferSize());
-                }
-                else
-                {
-                    elementStrings = ParseCSVElementStrings(args.CsvPath());
-                    WriteDataToBinding<uint16_t>(elementStrings, binding);
-                }
-                return TensorUInt16Bit::CreateFromArray(binding.GetShapeBuffer(), binding.GetDataBuffer());
+                return CreateTensor<TensorKind::UInt16>(args, elementStrings, tensorDescriptor);
             }
             break;
             case TensorKind::Int32:
             {
-                ModelBinding<int32_t> binding(description);
-                if (args.IsGarbageInput())
-                {
-                    memset(binding.GetData(), 0, sizeof(int32_t) * binding.GetDataBufferSize());
-                }
-                else
-                {
-                    elementStrings = ParseCSVElementStrings(args.CsvPath());
-                    WriteDataToBinding<int32_t>(elementStrings, binding);
-                }
-                return TensorInt32Bit::CreateFromArray(binding.GetShapeBuffer(), binding.GetDataBuffer());
+                return CreateTensor<TensorKind::Int32>(args, elementStrings, tensorDescriptor);
             }
             break;
             case TensorKind::UInt32:
             {
-                ModelBinding<uint32_t> binding(description);
-                if (args.IsGarbageInput())
-                {
-                    memset(binding.GetData(), 0, sizeof(uint32_t) * binding.GetDataBufferSize());
-                }
-                else
-                {
-                    elementStrings = ParseCSVElementStrings(args.CsvPath());
-                    WriteDataToBinding<uint32_t>(elementStrings, binding);
-                }
-                return TensorUInt32Bit::CreateFromArray(binding.GetShapeBuffer(), binding.GetDataBuffer());
+                return CreateTensor<TensorKind::UInt32>(args, elementStrings, tensorDescriptor);
             }
             break;
             case TensorKind::Int64:
             {
-                ModelBinding<int64_t> binding(description);
-                if (args.IsGarbageInput())
-                {
-                    memset(binding.GetData(), 0, sizeof(int64_t) * binding.GetDataBufferSize());
-                }
-                else
-                {
-                    elementStrings = ParseCSVElementStrings(args.CsvPath());
-                    WriteDataToBinding<int64_t>(elementStrings, binding);
-                }
-                return TensorInt64Bit::CreateFromArray(binding.GetShapeBuffer(), binding.GetDataBuffer());
+                return CreateTensor<TensorKind::Int64>(args, elementStrings, tensorDescriptor);
             }
             break;
             case TensorKind::UInt64:
             {
-                ModelBinding<uint64_t> binding(description);
-                if (args.IsGarbageInput())
-                {
-                    memset(binding.GetData(), 0, sizeof(uint64_t) * binding.GetDataBufferSize());
-                }
-                else
-                {
-                    elementStrings = ParseCSVElementStrings(args.CsvPath());
-                    WriteDataToBinding<uint64_t>(elementStrings, binding);
-                }
-                return TensorUInt64Bit::CreateFromArray(binding.GetShapeBuffer(), binding.GetDataBuffer());
+                return CreateTensor<TensorKind::UInt64>(args, elementStrings, tensorDescriptor);
             }
             break;
         }