Update Squeezenet c++ and C# sample to be explicit with colorspace management of GetSoftwareBitmapAsync (#282)

Ryan Lai · web-flow · commit e823735e66d8 · 2019-10-16T13:29:00.000-07:00
* Updated squeezenet cpp sample to be explicit about managing color space

* Updated squeezenet c# sample to be explicit about managing color space

* Case insensitive value when looking up metadata

* PR comments to make try catch tighter
diff --git a/Samples/SqueezeNetObjectDetection/Desktop/cpp/main.cpp b/Samples/SqueezeNetObjectDetection/Desktop/cpp/main.cpp
@@ -22,9 +22,10 @@ hstring imagePath;
 // helper functions
 string GetModulePath();
 void LoadLabels();
-VideoFrame LoadImageFile(hstring filePath);
+VideoFrame LoadImageFile(hstring filePath, ColorManagementMode colorManagementMode);
 void PrintResults(IVectorView<float> results);
 bool ParseArgs(int argc, char* argv[]);
+ColorManagementMode GetColorManagementMode(const LearningModel& model);
 
 wstring GetModelPath()
 {
@@ -57,9 +58,13 @@ int main(int argc, char* argv[])
     ticks = GetTickCount() - ticks;
     printf("model file loaded in %d ticks\n", ticks);
 
+    // get model color management mode
+    printf("Getting model color management mode...\n");
+    ColorManagementMode colorManagementMode = GetColorManagementMode(model);
+
     // load the image
     printf("Loading the image...\n");
-    auto imageFrame = LoadImageFile(imagePath);
+    auto imageFrame = LoadImageFile(imagePath, colorManagementMode);
     // now create a session and binding
     LearningModelSession session(model, LearningModelDevice(deviceKind));
     LearningModelBinding binding(session);
@@ -153,28 +158,76 @@ void LoadLabels()
     }
 }
 
-VideoFrame LoadImageFile(hstring filePath)
+ColorManagementMode GetColorManagementMode(const LearningModel& model)
+{
+    // Get model color space gamma
+    hstring gammaSpace = L"";
+    try
+    {
+        gammaSpace = model.Metadata().Lookup(L"Image.ColorSpaceGamma");
+    }
+    catch (...)
+    {
+        printf("    Model does not have color space gamma information. Will color manage to sRGB by default...\n");
+    }
+    if (gammaSpace == L"" || _wcsicmp(gammaSpace.c_str(), L"SRGB") == 0)
+    {
+        return ColorManagementMode::ColorManageToSRgb;
+    }
+    // Due diligence should be done to make sure that the input image is within the model's colorspace. There are multiple non-sRGB color spaces.
+    printf("    Model metadata indicates that color gamma space is : %ws. Will not manage color space to sRGB...\n", gammaSpace.c_str());
+    return ColorManagementMode::DoNotColorManage;
+}
+
+VideoFrame LoadImageFile(hstring filePath, ColorManagementMode colorManagementMode)
 {
+    BitmapDecoder decoder = NULL;
     try
     {
         // open the file
         StorageFile file = StorageFile::GetFileFromPathAsync(filePath).get();
         // get a stream on it
         auto stream = file.OpenAsync(FileAccessMode::Read).get();
         // Create the decoder from the stream
-        BitmapDecoder decoder = BitmapDecoder::CreateAsync(stream).get();
-        // get the bitmap
-        SoftwareBitmap softwareBitmap = decoder.GetSoftwareBitmapAsync().get();
-        // load a videoframe from it
-        VideoFrame inputImage = VideoFrame::CreateWithSoftwareBitmap(softwareBitmap);
-        // all done
-        return inputImage;
+        decoder = BitmapDecoder::CreateAsync(stream).get();
     }
     catch (...)
     {
-        printf("failed to load the image file, make sure you are using fully qualified paths\r\n");
+        printf("    Failed to load the image file, make sure you are using fully qualified paths\r\n");
         exit(EXIT_FAILURE);
     }
+    SoftwareBitmap softwareBitmap = NULL;
+    try
+    {
+        softwareBitmap = decoder.GetSoftwareBitmapAsync(
+            decoder.BitmapPixelFormat(),
+            decoder.BitmapAlphaMode(),
+            BitmapTransform(),
+            ExifOrientationMode::RespectExifOrientation,
+            colorManagementMode
+        ).get();
+    }
+    catch (hresult_error hr)
+    {
+        printf("    Failed to create SoftwareBitmap! Please make sure that input image is within the model's colorspace.\n");
+        printf("    %ws\n", hr.message().c_str());
+        exit(hr.code());
+    }
+    VideoFrame inputImage = NULL;
+    try
+    {
+        // load a videoframe from it
+        inputImage = VideoFrame::CreateWithSoftwareBitmap(softwareBitmap);
+    }
+    catch (hresult_error hr)
+    {
+        printf("Failed to create videoframe from software bitmap.");
+        printf("    %ws\n", hr.message().c_str());
+        exit(hr.code());
+    }
+    // all done
+    return inputImage;
+    
 }
 
 void PrintResults(IVectorView<float> results)
diff --git a/Samples/SqueezeNetObjectDetection/NETCore/cs/Program.cs b/Samples/SqueezeNetObjectDetection/NETCore/cs/Program.cs
@@ -45,8 +45,11 @@ static int Main(string[] args)
             // Create the evaluation session with the model and device
             _session = new LearningModelSession(_model, new LearningModelDevice(_deviceKind));
 
+            Console.WriteLine("Getting color management mode...");
+            ColorManagementMode colorManagementMode = GetColorManagementMode();
+
             Console.WriteLine("Loading the image...");
-            ImageFeatureValue imageTensor = LoadImageFile();
+            ImageFeatureValue imageTensor = LoadImageFile(colorManagementMode);
 
             // create a binding object from the session
             Console.WriteLine("Binding...");
@@ -117,17 +120,63 @@ private static T AsyncHelper<T> (IAsyncOperation<T> operation)
             return operation.GetResults();
         }
 
-        private static ImageFeatureValue LoadImageFile()
+        private static ImageFeatureValue LoadImageFile(ColorManagementMode colorManagementMode)
         {
-            StorageFile imageFile = AsyncHelper(StorageFile.GetFileFromPathAsync(_imagePath));
-            IRandomAccessStream stream = AsyncHelper(imageFile.OpenReadAsync());
-            BitmapDecoder decoder = AsyncHelper(BitmapDecoder.CreateAsync(stream));
-            SoftwareBitmap softwareBitmap = AsyncHelper(decoder.GetSoftwareBitmapAsync());
+            BitmapDecoder decoder = null;
+            try
+            {
+                StorageFile imageFile = AsyncHelper(StorageFile.GetFileFromPathAsync(_imagePath));
+                IRandomAccessStream stream = AsyncHelper(imageFile.OpenReadAsync());
+                decoder = AsyncHelper(BitmapDecoder.CreateAsync(stream));
+            }
+            catch (Exception e)
+            {
+                Console.WriteLine("Failed to load image file! Make sure that fully qualified paths are used.");
+                Console.WriteLine(" Exception caught.\n {0}", e);
+                System.Environment.Exit(e.HResult);
+            }
+            SoftwareBitmap softwareBitmap = null;
+            try
+            {
+                softwareBitmap = AsyncHelper(
+                    decoder.GetSoftwareBitmapAsync(
+                        decoder.BitmapPixelFormat,
+                        decoder.BitmapAlphaMode,
+                        new BitmapTransform(),
+                        ExifOrientationMode.RespectExifOrientation,
+                        colorManagementMode
+                    )
+                );
+            }
+            catch (Exception e)
+            {
+                Console.WriteLine("Failed to create SoftwareBitmap! Please make sure that input image is within the model's colorspace.");
+                Console.WriteLine(" Exception caught.\n {0}", e);
+                System.Environment.Exit(e.HResult);
+            }
             softwareBitmap = SoftwareBitmap.Convert(softwareBitmap, BitmapPixelFormat.Bgra8, BitmapAlphaMode.Premultiplied);
             VideoFrame inputImage = VideoFrame.CreateWithSoftwareBitmap(softwareBitmap);
             return ImageFeatureValue.CreateFromVideoFrame(inputImage);
         }
 
+        private static ColorManagementMode GetColorManagementMode()
+        {
+            // Get model color space gamma
+            string gammaSpace = "";
+            bool doesModelContainGammaSpaceMetadata = _model.Metadata.TryGetValue("Image.ColorSpaceGamma", out gammaSpace);
+            if (!doesModelContainGammaSpaceMetadata)
+            {
+                Console.WriteLine("    Model does not have color space gamma information. Will color manage to sRGB by default...");
+            }
+            if (!doesModelContainGammaSpaceMetadata || gammaSpace.Equals("SRGB", StringComparison.CurrentCultureIgnoreCase))
+            {
+                return ColorManagementMode.ColorManageToSRgb;
+            }
+            // Due diligence should be done to make sure that the input image is within the model's colorspace. There are multiple non-sRGB color spaces.
+            Console.WriteLine("    Model metadata indicates that color gamma space is : {0}. Will not manage color space to sRGB...", gammaSpace);
+            return ColorManagementMode.DoNotColorManage;
+        }
+
         private static void PrintResults(IReadOnlyList<float> resultVector)
         {
             // load the labels
