Merge pull request #429 from microsoft/user/sheilk/opencv-test

Add OpenCV Interop sample to the gallery

Author: smk2007

.gitignore

@@ -13,12 +13,14 @@
 *.userprefs
 
 # Build results
+[Bb]uild/
 [Dd]ebug/
 [Dd]ebugPublic/
 [Rr]elease/
 [Rr]eleases/
 x64/
 x86/
+[Ii]nt/
 bld/
 [Bb]in/
 [Oo]bj/

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "Samples/RustSqueezenet/winrt-rs"]
 	path = Samples/RustSqueezenet/winrt-rs
 	url = https://github.com/microsoft/winrt-rs.git
+[submodule "external/opencv"]
+	path = external/opencv
+	url = https://github.com/opencv/opencv.git

README.md

@@ -54,24 +54,24 @@ A subdomain of computer vision in which an algorithm looks at an image and assig
 
 | Windows App Type <br/>Distribution | UWP<br/>In-Box |  UWP<br/>NuGet | Desktop<br/>In-Box | Desktop<br/>NuGet |
 |------------|------------------------------------|--------------------------------------|------------------------------------|--------------------------------------|
-| [AlexNet](https://github.com/onnx/models/tree/master/vision/classification/alexnet)                                | | ||[✔️C# .NET5 - Samples Gallery](https://github.com/microsoft/Windows-Machine-Learning/tree/master/Samples/WinMLSamplesGallery)<br/>|
-| [CaffeNet](https://github.com/onnx/models/tree/master/vision/classification/caffenet)                                  | | ||[✔️C# .NET5 - Samples Gallery](https://github.com/microsoft/Windows-Machine-Learning/tree/master/Samples/WinMLSamplesGallery)<br/>|
-| [DenseNet](https://github.com/onnx/models/tree/master/vision/classification/densenet-121)                              | | ||[✔️C# .NET5 - Samples Gallery](https://github.com/microsoft/Windows-Machine-Learning/tree/master/Samples/WinMLSamplesGallery)<br/>|
-| [EfficientNet](https://github.com/onnx/models/tree/master/vision/classification/efficientnet-lite4)             | | ||[✔️C# .NET5 - Samples Gallery](https://github.com/microsoft/Windows-Machine-Learning/tree/master/Samples/WinMLSamplesGallery)<br/>|
+| [AlexNet](https://github.com/onnx/models/tree/master/vision/classification/alexnet)                                | | ||[✔️C# .NET5 - Samples Gallery](Samples/WinMLSamplesGallery)<br/>|
+| [CaffeNet](https://github.com/onnx/models/tree/master/vision/classification/caffenet)                                  | | ||[✔️C# .NET5 - Samples Gallery](Samples/WinMLSamplesGallery)<br/>|
+| [DenseNet](https://github.com/onnx/models/tree/master/vision/classification/densenet-121)                              | | ||[✔️C# .NET5 - Samples Gallery](Samples/WinMLSamplesGallery)<br/>|
+| [EfficientNet](https://github.com/onnx/models/tree/master/vision/classification/efficientnet-lite4)             | | ||[✔️C# .NET5 - Samples Gallery](Samples/WinMLSamplesGallery)<br/>|
 | [Emoji8](https://blogs.windows.com/windowsdeveloper/2018/11/16/introducing-emoji8/)     | [✔️C#](https://github.com/microsoft/Windows-Machine-Learning/tree/master/Samples/Emoji8/UWP/cs) |                                                 |
-| [GoogleNet](https://github.com/onnx/models/tree/master/vision/classification/inception_and_googlenet/googlenet)        | | ||[✔️C# .NET5 - Samples Gallery](https://github.com/microsoft/Windows-Machine-Learning/tree/master/Samples/WinMLSamplesGallery)<br/>|
-| [InceptionV1](https://github.com/onnx/models/tree/master/vision/classification/inception_and_googlenet/inception_v1)      | | ||[✔️C# .NET5 - Samples Gallery](https://github.com/microsoft/Windows-Machine-Learning/tree/master/Samples/WinMLSamplesGallery)<br/>|
-| [InceptionV2](https://github.com/onnx/models/tree/master/vision/classification/inception_and_googlenet/inception_v2)      | | ||[✔️C# .NET5 - Samples Gallery](https://github.com/microsoft/Windows-Machine-Learning/tree/master/Samples/WinMLSamplesGallery)<br/>|
+| [GoogleNet](https://github.com/onnx/models/tree/master/vision/classification/inception_and_googlenet/googlenet)        | | ||[✔️C# .NET5 - Samples Gallery](Samples/WinMLSamplesGallery)<br/>|
+| [InceptionV1](https://github.com/onnx/models/tree/master/vision/classification/inception_and_googlenet/inception_v1)      | | ||[✔️C# .NET5 - Samples Gallery](Samples/WinMLSamplesGallery)<br/>|
+| [InceptionV2](https://github.com/onnx/models/tree/master/vision/classification/inception_and_googlenet/inception_v2)      | | ||[✔️C# .NET5 - Samples Gallery](Samples/WinMLSamplesGallery)<br/>|
 | [MNIST](https://github.com/onnx/models/tree/master/vision/classification/mnist)      | [✔️C++/CX](https://github.com/Microsoft/Windows-Machine-Learning/tree/master/Samples/MNIST/UWP)<br/>[✔️C#](https://github.com/Microsoft/Windows-Machine-Learning/tree/master/Samples/MNIST/Tutorial/cs)<br/>    |                                                 |
-| [MobileNetV2](https://github.com/onnx/models/tree/master/vision/classification/mobilenet)                              | | ||[✔️C# .NET5 - Samples Gallery](https://github.com/microsoft/Windows-Machine-Learning/tree/master/Samples/WinMLSamplesGallery)<br/>|
-| [RCNN](https://github.com/onnx/models/tree/master/vision/classification/rcnn_ilsvrc13)                        | | ||[✔️C# .NET5 - Samples Gallery](https://github.com/microsoft/Windows-Machine-Learning/tree/master/Samples/WinMLSamplesGallery)<br/>|
-| [ResNet50](https://github.com/onnx/models/tree/master/vision/classification/resnet)                          | | ||[✔️C# .NET5 - Samples Gallery](https://github.com/microsoft/Windows-Machine-Learning/tree/master/Samples/WinMLSamplesGallery)<br/>|
-| [ShuffleNetV1](https://github.com/onnx/models/tree/master/vision/classification/shufflenet)                              | | ||[✔️C# .NET5 - Samples Gallery](https://github.com/microsoft/Windows-Machine-Learning/tree/master/Samples/WinMLSamplesGallery)<br/>|
-| [ShuffleNetV2](https://github.com/onnx/models/tree/master/vision/classification/shufflenet)                          | | ||[✔️C# .NET5 - Samples Gallery](https://github.com/microsoft/Windows-Machine-Learning/tree/master/Samples/WinMLSamplesGallery)<br/>|
-| [SqueezeNet](https://github.com/onnx/models/tree/master/vision/classification/squeezenet) | [✔️C#](https://github.com/Microsoft/Windows-Machine-Learning/tree/master/Samples/SqueezeNetObjectDetection/UWP/cs)<br/>[✔️JavaScript](https://github.com/Microsoft/Windows-Machine-Learning/tree/master/Samples/SqueezeNetObjectDetection/UWP/cs)<br/>        |                     |[✔️C++/WinRT](https://github.com/Microsoft/Windows-Machine-Learning/tree/master/Samples/SqueezeNetObjectDetection/Desktop/cpp)<br/> [✔️C# .NET5](https://github.com/Microsoft/Windows-Machine-Learning/tree/master/Samples/SqueezeNetObjectDetection/NET5)<br/>[✔️C# .NET Core 2](https://github.com/microsoft/Windows-Machine-Learning/tree/master/Samples/SqueezeNetObjectDetection/NETCore/cs)<br/>|[✔️C++/WinRT](https://github.com/Microsoft/Windows-Machine-Learning/tree/master/Samples/SqueezeNetObjectDetection/Desktop/cpp)<br/>[✔️C# .NET5 - Samples Gallery](https://github.com/microsoft/Windows-Machine-Learning/tree/master/Samples/WinMLSamplesGallery)<br/>[✔️Rust](https://github.com/microsoft/Windows-Machine-Learning/tree/master/Samples/RustSqueezenet)<br/>|
-| [VGG19](https://github.com/onnx/models/tree/master/vision/classification/vgg)                                          | | ||[✔️C# .NET5 - Samples Gallery](https://github.com/microsoft/Windows-Machine-Learning/tree/master/Samples/WinMLSamplesGallery)<br/>|
-| [VGG19bn](https://github.com/onnx/models/tree/master/vision/classification/vgg)                                       | | ||[✔️C# .NET5 - Samples Gallery](https://github.com/microsoft/Windows-Machine-Learning/tree/master/Samples/WinMLSamplesGallery)<br/>|
-| [ZFNet512](https://github.com/onnx/models/tree/master/vision/classification/zfnet-512)                                 | | ||[✔️C# .NET5 - Samples Gallery](https://github.com/microsoft/Windows-Machine-Learning/tree/master/Samples/WinMLSamplesGallery)<br/>|
+| [MobileNetV2](https://github.com/onnx/models/tree/master/vision/classification/mobilenet)                              | | ||[✔️C# .NET5 - Samples Gallery](Samples/WinMLSamplesGallery)<br/>|
+| [RCNN](https://github.com/onnx/models/tree/master/vision/classification/rcnn_ilsvrc13)                        | | ||[✔️C# .NET5 - Samples Gallery](Samples/WinMLSamplesGallery)<br/>|
+| [ResNet50](https://github.com/onnx/models/tree/master/vision/classification/resnet)                          | | ||[✔️C# .NET5 - Samples Gallery](Samples/WinMLSamplesGallery)<br/>|
+| [ShuffleNetV1](https://github.com/onnx/models/tree/master/vision/classification/shufflenet)                              | | ||[✔️C# .NET5 - Samples Gallery](Samples/WinMLSamplesGallery)<br/>|
+| [ShuffleNetV2](https://github.com/onnx/models/tree/master/vision/classification/shufflenet)                          | | ||[✔️C# .NET5 - Samples Gallery](Samples/WinMLSamplesGallery)<br/>|
+| [SqueezeNet](https://github.com/onnx/models/tree/master/vision/classification/squeezenet) | [✔️C#](https://github.com/Microsoft/Windows-Machine-Learning/tree/master/Samples/SqueezeNetObjectDetection/UWP/cs)<br/>[✔️JavaScript](https://github.com/Microsoft/Windows-Machine-Learning/tree/master/Samples/SqueezeNetObjectDetection/UWP/cs)<br/>        |                     |[✔️C++/WinRT](https://github.com/Microsoft/Windows-Machine-Learning/tree/master/Samples/SqueezeNetObjectDetection/Desktop/cpp)<br/> [✔️C# .NET5](https://github.com/Microsoft/Windows-Machine-Learning/tree/master/Samples/SqueezeNetObjectDetection/NET5)<br/>[✔️C# .NET Core 2](https://github.com/microsoft/Windows-Machine-Learning/tree/master/Samples/SqueezeNetObjectDetection/NETCore/cs)<br/>|[✔️C++/WinRT](https://github.com/Microsoft/Windows-Machine-Learning/tree/master/Samples/SqueezeNetObjectDetection/Desktop/cpp)<br/>[✔️C# .NET5 - Samples Gallery](Samples/WinMLSamplesGallery)<br/>[✔️Rust](https://github.com/microsoft/Windows-Machine-Learning/tree/master/Samples/RustSqueezenet)<br/>|
+| [VGG19](https://github.com/onnx/models/tree/master/vision/classification/vgg)                                          | | ||[✔️C# .NET5 - Samples Gallery](Samples/WinMLSamplesGallery)<br/>|
+| [VGG19bn](https://github.com/onnx/models/tree/master/vision/classification/vgg)                                       | | ||[✔️C# .NET5 - Samples Gallery](Samples/WinMLSamplesGallery)<br/>|
+| [ZFNet512](https://github.com/onnx/models/tree/master/vision/classification/zfnet-512)                                 | | ||[✔️C# .NET5 - Samples Gallery](Samples/WinMLSamplesGallery)<br/>|
 
 **Style Transfer**
 
@@ -87,7 +87,7 @@ A computer vision technique that allows us to recompose the content of an image
 
 |            | Store App<br/>Inbox API |  Store App<br/>NuGet API | Desktop App<br/>Inbox API |  Desktop App<br/>NuGet API |
 |------------|------------------------------------|--------------------------------------|------------------------------------|--------------------------------------|
-| [YoloV4](https://github.com/onnx/models/raw/master/vision/object_detection_segmentation/yolov4/model/yolov4.onnx)                         | | ||[✔️C# .NET5 - Samples Gallery](https://github.com/microsoft/Windows-Machine-Learning/tree/master/Samples/WinMLSamplesGallery)<br/>|
+| [YoloV4](https://github.com/onnx/models/raw/master/vision/object_detection_segmentation/yolov4/model/yolov4.onnx)                         | | ||[✔️C# .NET5 - Samples Gallery](Samples/WinMLSamplesGallery)<br/>|
 
 -->
 
@@ -104,6 +104,7 @@ These advanced samples show how to use various binding and evaluation features i
 - **[PyTorch Data Analysis](https://github.com/Microsoft/Windows-AppConsult-Samples-UWP/tree/master/PlaneIdentifier)**: The tutorial shows how to solve a classification task with a neural network using the PyTorch library, export the model to ONNX format and deploy the model with the Windows Machine Learning application that can run on any Windows device.
 - **[PyTorch Image Classification](https://github.com/Microsoft/Windows-AppConsult-Samples-UWP/tree/master/PlaneIdentifier)**: The tutorial shows how to train an image classification neural network model using PyTorch, export the model to the ONNX format, and deploy it in a Windows Machine Learning application running locally on your Windows device.
 - **[YoloV4 Object Detection](https://github.com/Microsoft/Windows-AppConsult-Samples-UWP/tree/master/PlaneIdentifier)**: This tutorial shows how to build a UWP C# app that uses the YOLOv4 model to detect objects in video streams.
+- **[OpenCV Interop](Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/OpenCVInterop)**: This sample demonstrates how to interop between [Windows ML](https://docs.microsoft.com/en-us/windows/ai/windows-ml/) and [OpenCV](https://github.com/opencv/opencv).
 
 ## Developer Tools
 
@@ -135,7 +136,7 @@ These advanced samples show how to use various binding and evaluation features i
 
     Download for [VS 2017](https://marketplace.visualstudio.com/items?itemName=WinML.mlgen), [VS 2019](https://marketplace.visualstudio.com/items?itemName=WinML.MLGenV2)
 
-  - **[WinML Samples Gallery](https://github.com/microsoft/Windows-Machine-Learning/tree/master/Samples/WinMLSamplesGallery):** explore a variety of ML integration scenarios and models.
+  - **[WinML Samples Gallery](Samples/WinMLSamplesGallery):** explore a variety of ML integration scenarios and models.
 
   - Check out the [Model Samples](#model-samples) and [Advanced Scenario Samples](#advanced-scenarios) to learn how to use Windows ML in your application.
 

Samples/WinMLSamplesGallery/README.md

@@ -36,6 +36,8 @@ To learn how to implement these features in your application, or unlock addition
 
 - [Batched Inputs](./WinMLSamplesGallery/Samples/Batching): WinML enables batched inputs that allow callers to perform inference over multiple inputs at once in order to increase performance. Use this sample to compare inference runtime performace with and without batching.
 
+- [OpenCV Interop](./WinMLSamplesGallery/Samples/OpenCVInterop): This sample demonstrates how to interop between [Windows ML](https://docs.microsoft.com/en-us/windows/ai/windows-ml/) and [OpenCV](https://github.com/opencv/opencv). The demo will run [SqueezeNet](https://github.com/onnx/models/tree/master/vision/classification/squeezenet) image classification in WindowsML and consume images loaded and preprocessed using OpenCV.
+
 ## Feedback
 Please file an issue [here](https://github.com/microsoft/Windows-Machine-Learning/issues/new) if you encounter any issues with the WinML Samples Gallery or wish to request a new sample.
 

Samples/WinMLSamplesGallery/WinMLSamplesGallery (Package)/WinMLSamplesGallery (Package).wapproj

@@ -128,6 +128,7 @@
   </ItemGroup>
   <ItemGroup>
     <None Include="$(SolutionDir)$(Platform)\$(Configuration)\WinMLSamplesGalleryNative\*.dll" Link="%(RecursiveDir)%(Filename)%(Extension)" CopyToOutputDirectory="PreserveNewest" />
+    <None Include="$(SolutionDir)..\..\build\external\opencv\cmake_config\$(Platform)\bin\$(Configuration)\*.dll" Link="%(RecursiveDir)%(Filename)%(Extension)" CopyToOutputDirectory="PreserveNewest" />
   </ItemGroup>
   <Import Project="$(WapProjPath)\Microsoft.DesktopBridge.targets" />
   <ItemGroup>

Samples/WinMLSamplesGallery/WinMLSamplesGallery/Pages/SampleBasePage.xaml.cs

@@ -28,6 +28,9 @@ protected override async void OnNavigatedTo(NavigationEventArgs e)
                 case "ImageEffects":
                     SampleFrame.Navigate(typeof(Samples.ImageEffects));
                     break;
+                case "OpenCVInterop":
+                    SampleFrame.Navigate(typeof(Samples.OpenCVInterop));
+                    break;
             }
             if (sampleMetadata.Docs.Count > 0)
                 DocsHeader.Visibility = Visibility.Visible;

Samples/WinMLSamplesGallery/WinMLSamplesGallery/SampleMetadata/SampleMetadata.cs

@@ -39,6 +39,17 @@ public static async Task<List<SampleMetadata>> GetAllSampleMetadata()
             for (int i = 0; i < metadataJsonArray.Count; i++)
             {
                 JsonObject currentSampleMetadata = metadataJsonArray[i].GetObject();
+
+                bool shouldHideSample = false;
+#if !USE_OPENCV
+                shouldHideSample |= currentSampleMetadata["Tag"].GetString() == "OpenCVInterop";
+#endif
+
+                if (shouldHideSample)
+                {
+                    continue;
+                }
+
                 allSampleMetadata.Add(new SampleMetadata
                 {
                     Title = currentSampleMetadata["Title"].GetString(),

Samples/WinMLSamplesGallery/WinMLSamplesGallery/SampleMetadata/SampleMetadata.json

@@ -34,6 +34,16 @@
           "link": "https://docs.microsoft.com/en-us/uwp/api/windows.ai.machinelearning.learningmodelsessionoptions.batchsizeoverride?view=winrt-20348"
         }
       ]
+    },
+    {
+      "Title": "OpenCV Interop",
+      "DescriptionShort": "The sample uses Windows ML to classify images that have been denoised using OpenCV.",
+      "Description": "This sample demonstrates interop between Windows ML and OpenCV. The sample classifes images that have been denoised using OpenCV's medianBlur using the SqueezeNet model on Windows ML. Choose an image to get started.",
+      "Icon": "\uE155",
+      "Tag": "OpenCVInterop",
+      "XAMLGithubLink": "https://github.com/microsoft/Windows-Machine-Learning/blob/master/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/OpenCVInterop/OpenCVInterop.xaml",
+      "CSharpGithubLink": "https://github.com/microsoft/Windows-Machine-Learning/blob/master/Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/OpenCVInterop/OpenCVInterop.xaml.cs",
+      "Docs": []
     }
   ]
 }

Samples/WinMLSamplesGallery/WinMLSamplesGallery/Samples/ImageClassifier/ImageClassifier.xaml.cs

@@ -233,13 +233,11 @@ private void EnsureInitialized()
 
         private void InitializeWindowsMachineLearning()
         {
-            _tensorizationSession =
-                CreateLearningModelSession(
-                    TensorizationModels.BasicTensorization(
-                        Height, Width,
-                        BatchSize, Channels, CurrentImageDecoder.PixelHeight, CurrentImageDecoder.PixelWidth,
-                        "nearest"),
-                        LearningModelDeviceKind.Cpu);
+            _tensorizationSession = CreateLearningModelSession(TensorizationModels.BasicTensorization(
+                Height, Width,
+                BatchSize, Channels, CurrentImageDecoder.PixelHeight, CurrentImageDecoder.PixelWidth,
+                "nearest"),
+                LearningModelDeviceKind.Cpu);
 
             var model = SelectedModel;
             if (model != CurrentModel)
@@ -287,7 +285,7 @@ private void InitializeWindowsMachineLearning()
             {
                 var pixelDataProvider = decoder.GetPixelDataAsync().GetAwaiter().GetResult();
                 var bytes = pixelDataProvider.DetachPixelData();
-                var buffer = bytes.AsBuffer(); // Need to make this 0 copy...
+                var buffer = bytes.AsBuffer();
                 input = TensorUInt8Bit.CreateFromBuffer(new long[] { 1, buffer.Length }, buffer);
 
                 tensorizationSession = _tensorizationSession;
@@ -303,6 +301,7 @@ private void InitializeWindowsMachineLearning()
                 var tensorizationResults = Evaluate(tensorizationSession, input);
                 tensorizedOutput = tensorizationResults.Outputs.First().Value;
             }
+
             stop = HighResolutionClock.UtcNow();
             var tensorizeDuration = HighResolutionClock.DurationInMs(start, stop);