Different results from same model on C# and Python inferences

[karlhm76]

Hello,

I have trained a YOLOv5 model using custom dataset and exported to ONNX.

It took a little while to discover the idiosyncrasies of the YOLOv5 model output but I have it working now in Python and performance is very good, however I need to have it working in C#.

Even though I am using the same ORT version, the same model and the same image input I am not getting quality results in C#.

The problem must be something simple with image input or with result interpretation, but I have not been able to discover what that is.

My C# code is here, I have been trying multiple things:

        private void cmdAnalyse_Click(object sender, EventArgs e)
        {
            // begin analysis
            if (this.txtONNXFile.Text == "")
            {
                MessageBox.Show("Please select an ONNX file");
                return;
            }

            if (this.originalImage != null)
            {
                this.originalImage.Dispose();
                this.originalImage = new Bitmap(this.txtImageFile.Text);
            }

            if (this.originalImage == null)
            {
                if (this.txtImageFile.Text == "")
                    MessageBox.Show("Please select an image");
                return;

                this.originalImage = new Bitmap(this.txtImageFile.Text);
            }


            // Resize the image to 640x640
            int desiredWidth = 640;
            int desiredHeight = 640;

            float scaleX = (float)originalImage.Width / desiredWidth;
            float scaleY = (float)originalImage.Height / desiredHeight;

            Bitmap resizedImage = new Bitmap(desiredWidth, desiredHeight);
            Rectangle imageRect = new Rectangle(0, 0, desiredWidth, desiredHeight);

            using (Graphics g = Graphics.FromImage(resizedImage))
            {
                g.InterpolationMode = System.Drawing.Drawing2D.InterpolationMode.HighQualityBicubic; // Smoother interpolation for downscaling
                g.DrawImage(originalImage, imageRect);
                this.pbRibeye.Image = resizedImage;
            }


            Microsoft.ML.OnnxRuntime.Tensors.Tensor<float> input = new Microsoft.ML.OnnxRuntime.Tensors.DenseTensor<float>(new[] { 1, 3, desiredWidth, desiredHeight });

            BitmapData bitmapData = resizedImage.LockBits(imageRect, ImageLockMode.ReadOnly, PixelFormat.Format24bppRgb);

            int stride = bitmapData.Stride;
            IntPtr scan0 = bitmapData.Scan0;

            unsafe
            {
                byte* ptr = (byte*)scan0;
                for (int img_y = 0; img_y < resizedImage.Height; img_y++)
                {
                    for (int img_x = 0; img_x < resizedImage.Width; img_x++)
                    {
                        int offset = img_y * stride + img_x * 3;
                        input[0, 0, img_y, img_x] = ptr[offset + 2]; // Red channel
                        input[0, 1, img_y, img_x] = ptr[offset + 1]; // Green channel
                        input[0, 2, img_y, img_x] = ptr[offset];     // Blue channel
                    }
                }
            }


            resizedImage.UnlockBits(bitmapData);


            var inputs = new List<Microsoft.ML.OnnxRuntime.NamedOnnxValue>
            {
                // this is the name of the input node, found using Netron
                Microsoft.ML.OnnxRuntime.NamedOnnxValue.CreateFromTensor("images", input)
            };

            // run inference

            var session = new Microsoft.ML.OnnxRuntime.InferenceSession(this.txtONNXFile.Text);
            Microsoft.ML.OnnxRuntime.IDisposableReadOnlyCollection<Microsoft.ML.OnnxRuntime.DisposableNamedOnnxValue> results = session.Run(inputs);


            // Extract the output tensor from the results
            var outputTensor = results.FirstOrDefault().AsTensor<float>();

            var shape = outputTensor.Dimensions.ToArray();
            int batchSize = shape[0];
            int numPredictions = shape[1];

            int maxConfidenceIndex = 0;
            float maxConf = 0;

            for (int p = 0; p < numPredictions; p++)
            {
                float conf = Sigmoid(outputTensor[0, p, 4]);
                if (conf > maxConf)
                {
                    maxConf = conf;
                    maxConfidenceIndex = p;
                }
            }

            float x = outputTensor[0, maxConfidenceIndex, 0];
            float y = outputTensor[0, maxConfidenceIndex, 1];


            // Get the box with the highest confidence
            float xCenter = outputTensor[0, maxConfidenceIndex, 0];
            float yCenter = outputTensor[0, maxConfidenceIndex, 1];
            float width = outputTensor[0, maxConfidenceIndex, 2];
            float height = outputTensor[0, maxConfidenceIndex, 3];

            float confidence = outputTensor[0, maxConfidenceIndex, 4];

            float xMin = x * scaleX;
            float yMin = y * scaleY;
            float xMax = (x + width ) * scaleX;
            float yMax = (y + height ) * scaleY;



            //float xMin = (xCenter - width / 2) * scaleX;
            //float yMin = (yCenter - height / 2) * scaleY;
            //float xMax = (xCenter + width / 2) * scaleX;
            //float yMax = (yCenter + height / 2) * scaleY;

            System.Drawing.Graphics graph = System.Drawing.Graphics.FromImage(this.pbRibeye.Image);

            // Put boxes, labels and confidence on image and save for viewing

            System.Drawing.Pen pen = new System.Drawing.Pen(System.Drawing.Color.Red, 2);

            //int x1 = (int)((x - w / 2) * scaleX);
            //int y1 = (int)((y - h / 2) * scaleY);
            //int x2 = (int)((x + w / 2) * scaleX);
            //int y2 = (int)((y + h / 2) * scaleY);

            // Draw the bounding box on the original image
            graph.DrawRectangle(pen, xMin, yMin, xMax - xMin, yMax - yMin);

            // Create a rectangle to draw the label text
            RectangleF rectf = new RectangleF(xMin, yMin - 12, xMax - xMin, 10);

            StringFormat format = new StringFormat()
            {
                Alignment = StringAlignment.Near,
                LineAlignment = StringAlignment.Near
            };

            // Draw the text onto the image, showing the label and confidence score
            graph.DrawString((confidence * 100).ToString() + "%", new Font("Tahoma", 8), Brushes.Red, rectf, format);


            graph.Flush();
            graph.Dispose();

            //this.pbRibeye.Image = this.pbRibeye.Image;

        }

Thank you for any help

[karlhm76]

I have discovered the problem. There was 2 problems, The first problem was that the image pixels need to be normalised to values between 0 and 1 by /255f.

The other problem was that the confidences don't need to be scaled with sigmoid. As soon as I took that out of the C# program started working the same as the Python program.

the final C# code was:

        private void cmdAnalyse_Click(object sender, EventArgs e)
        {
            // begin analysis
            if (this.txtONNXFile.Text == "")
            {
                MessageBox.Show("Please select an ONNX file");
                return;
            }

            if (this.originalImage != null)
            {
                this.originalImage.Dispose();
                this.originalImage = new Bitmap(this.txtImageFile.Text);
            }

            if (this.originalImage == null)
            {
                if (this.txtImageFile.Text == "")
                {
                    MessageBox.Show("Please select an image");
                    return;
                }

                this.originalImage = new Bitmap(this.txtImageFile.Text);
                
            }



            // Resize the image to 640x640
            // Resize the image to 640x640 while preserving the aspect ratio
            int desiredWidth = 640;
            int desiredHeight = 640;

            float scale = Math.Min((float)desiredWidth / originalImage.Width, (float)desiredHeight / originalImage.Height);
            int newWidth = (int)(originalImage.Width * scale);
            int newHeight = (int)(originalImage.Height * scale);

            float scaleX = (float)originalImage.Width / (float)newWidth;
            float scaleY = (float)originalImage.Height / (float)newHeight;

            Bitmap resizedImage = new Bitmap(newWidth, newWidth);
            Rectangle imageRect = new Rectangle(0, 0, newWidth, newHeight);

            using (Graphics g = Graphics.FromImage(resizedImage))
            {
                g.InterpolationMode = System.Drawing.Drawing2D.InterpolationMode.HighQualityBicubic;
                g.DrawImage(originalImage, imageRect);
            }

            Microsoft.ML.OnnxRuntime.Tensors.Tensor<float> input = new Microsoft.ML.OnnxRuntime.Tensors.DenseTensor<float>(new[] { 1, 3, desiredWidth, desiredHeight });


            BitmapData bitmapData = resizedImage.LockBits(imageRect, ImageLockMode.ReadOnly, PixelFormat.Format24bppRgb);

            int stride = bitmapData.Stride;
            IntPtr scan0 = bitmapData.Scan0;

            unsafe
            {
                byte* ptr = (byte*)scan0;
                for (int img_y = 0; img_y < newHeight; img_y++)
                {
                    for (int img_x = 0; img_x < newWidth; img_x++)
                    {
                        int offset = img_y * stride + img_x * 3;
                        input[0, 0, img_y, img_x] = (ptr[offset + 2] / 255f); // Red channel
                        input[0, 1, img_y, img_x] = (ptr[offset + 1] / 255f); // Green channel
                        input[0, 2, img_y, img_x] = (ptr[offset] / 255f);     // Blue channel
                    }
                }
            }


            resizedImage.UnlockBits(bitmapData);

            var inputs = new List<Microsoft.ML.OnnxRuntime.NamedOnnxValue>
            {
                // this is the name of the input node, found using Netron
                Microsoft.ML.OnnxRuntime.NamedOnnxValue.CreateFromTensor("images", input)
            };

            // run inference

            var session = new Microsoft.ML.OnnxRuntime.InferenceSession(this.txtONNXFile.Text);
            Microsoft.ML.OnnxRuntime.IDisposableReadOnlyCollection<Microsoft.ML.OnnxRuntime.DisposableNamedOnnxValue> results = session.Run(inputs);


            // Extract the output tensor from the results
            var outputTensor = results.FirstOrDefault().AsTensor<float>();

            var shape = outputTensor.Dimensions.ToArray();
            //int batchSize = shape[0];
            int numPredictions = shape[1];

            int maxConfidenceIndex = 0;
            float maxConf = 0;

            for (int p = numPredictions - 1; p >= 0; p--)
            {
                float conf = outputTensor[0, p, 4];
                if (conf > maxConf)
                {
                    maxConf = conf;
                    maxConfidenceIndex = p;

                    if (maxConf > 0.95) break;
                }
            }

            // Get the box with the highest confidence
            float xCenter = outputTensor[0, maxConfidenceIndex, 0];
            float yCenter = outputTensor[0, maxConfidenceIndex, 1];
            float width = outputTensor[0, maxConfidenceIndex, 2];
            float height = outputTensor[0, maxConfidenceIndex, 3];

            float confidence = outputTensor[0, maxConfidenceIndex, 4];

            // Calculate the inverse scaling factors
            float scaleXInv = (float)originalImage.Width / newWidth;
            float scaleYInv = (float)originalImage.Height / newHeight;

            // Scale the bounding box coordinates back to the original image dimensions
            float xMin = xCenter * scaleXInv - width * scaleXInv / 2;
            float yMin = yCenter * scaleYInv - height * scaleYInv / 2;
            float xMax = xCenter * scaleXInv + width * scaleXInv / 2;
            float yMax = yCenter * scaleYInv + height * scaleYInv / 2;

            // Draw the bounding box on the original image
            System.Drawing.Graphics graph = System.Drawing.Graphics.FromImage(this.originalImage);
            System.Drawing.Pen pen = new System.Drawing.Pen(System.Drawing.Color.Red, 2);
            graph.DrawRectangle(pen, xMin, yMin, xMax - xMin, yMax - yMin);

            // Create a rectangle to draw the label text
            RectangleF rectf = new RectangleF(xMin, yMin - 12, xMax - xMin, 10);

            StringFormat format = new StringFormat()
            {
                Alignment = StringAlignment.Near,
                LineAlignment = StringAlignment.Near
            };

            // Draw the text onto the image, showing the label and confidence score
            graph.DrawString("Ribeye " + (confidence * 100).ToString() + "%", new Font("Tahoma", 8), Brushes.Red, rectf, format);


            graph.Flush();
            graph.Dispose();

            this.pbRibeye.Image = this.originalImage;

        }