cpp inference result bad and is different python inference result, why?

[CsVeryLoveXieWenLi]

This is onnx file.

FastestDet2.zip

I use pypi onnxruntime1.12.1 and libonnxruntime.so.1.12.1

This is Object detection network

output shape is {object_num, 6}

The python result is success.

python code and output:

import cv2
import time
import onnxruntime

if __name__ == '__main__':
    # 读取图片
    img = cv2.imread("3.jpg")
    imgi = cv2.resize(img, (352, 352), interpolation=cv2.INTER_LINEAR)
    imgi = imgi.astype('float32') / 255
    H, W, C = img.shape
    # 模型输入的宽高
    input_width, input_height = 352, 352
    # 加载模型
    session = onnxruntime.InferenceSession('FastestDet2.onnx')
    # 目标检测
    start = time.perf_counter()
    out = session.run(None, {
        session.get_inputs()[0].name: [imgi.transpose(2, 0, 1)]
    })[0]
    end = time.perf_counter()
    time = (end - start) * 1000.
    print("forward time:%fms" % time)

    # 加载label names
    names = []
    with open("coco.names", 'r') as f:
        for line in f.readlines():
            names.append(line.strip())

    print(out.shape)
    print("=================box info===================")
    for b in out:
        print(b)
        obj_score, cls_index = b[4], int(b[5])
        x1, y1, x2, y2 = int(b[0] * W), int(b[1] * H), int(b[2] * W), int(b[3] * H)

        # 绘制检测框
        cv2.rectangle(img, (x1, y1), (x2, y2), (255, 255, 0), 2)
        cv2.putText(img, '%.2f' % obj_score, (x1, y1 - 5), 0, 0.7, (0, 255, 0), 2)
        cv2.putText(img, names[cls_index], (x1, y1 - 25), 0, 0.7, (0, 255, 0), 2)

    cv2.imwrite("result.jpg", img)

forward time:9.255000ms
(3, 6)
=================box info===================
[0.05656719 0.17865632 0.23565933 0.4428749  0.9743588  0.        ]
[0.43060577 0.         0.61302793 0.26438925 0.9500997  0.        ]
[0.6211234  0.00162221 0.8111357  0.2740757  0.94730824 0.        ]

The cpp result is fail.

cpp code and output:

// 目标检测
class FastestDetEnd2
{
public:
    /**
     * @description: 初始化目标检测
     * @param {char} *onnx_model_path FastestDet
     * @return {void}
     */
    void init(char *onnx_model_path)
    {
        // 启用所有优化 -> 设定推理线程
        this->ort_session_opt.SetGraphOptimizationLevel(ORT_ENABLE_ALL);
        this->ort_session_opt.SetIntraOpNumThreads(2);

        // 初始化Seession
        this->ort_session = new Session(this->ort_env, onnx_model_path, this->ort_session_opt);

        // 输入输出名称
        this->input_names.push_back(this->ort_session->GetInputName(0, this->ort_allocator));
        this->output_names.push_back(this->ort_session->GetOutputName(0, this->ort_allocator));
    }
    /**
     * @description: 目标检测
     * @param {Mat} &image_src
     * @return {vector<vector<int>>} [[x1,y1,w,h]]
     */
    vector<vector<int>> detect(Mat &image_src)
    {
        // 信息
        // int image_height = image_src.rows;
        // int image_width = image_src.cols;

        // 复制图片
        Mat image_submit;
        image_src.copyTo(image_submit);

        // 预处理
        resize(image_submit, image_submit, this->input_size, INTER_LINEAR);
        image_submit.convertTo(image_submit, CV_32FC3);
        this->hwc2chw2one(image_submit);

        // 初始化输入
        Ort::Value input_tensor = Ort::Value::CreateTensor<float>(this->ort_memory_info, image_submit.ptr<float>(), image_submit.total(), this->input_shape.data(), this->input_shape.size());

        // 推理 output (num, 6) [[x1,y1,x2,y2,score,class]]
        auto output_tensors = this->ort_session->Run(Ort::RunOptions{nullptr}, this->input_names.data(), &input_tensor, this->input_names.size(), this->output_names.data(), this->output_names.size());

        // 释放资源
        image_submit.release();

        // 获取结果
        int object_num = output_tensors[0].GetTensorTypeAndShapeInfo().GetShape()[0];
        int output_size = object_num * 6;
        float *output_tensor = output_tensors[0].GetTensorMutableData<float>();
        vector<float> output_data(output_tensor, output_tensor + output_size);
        printf("object_num %i\n", object_num);

        // 处理结果
        vector<vector<int>> box;
        vector<float>::iterator start;
        int x1, y1, x2, y2;

        for (int index = 0; index < output_size; index += 6)
        {
            start = output_data.begin() + index;
            x1 = int(start[0] * image_src.cols);
            y1 = int(start[1] * image_src.rows);
            x2 = int(start[2] * image_src.cols);
            y2 = int(start[3] * image_src.rows);
            // x,y,w,h
            box.push_back({x1, y1, x2 - x1, y2 - y1});
        }

        return box;
    }

private:
    // onnxruntime
    Ort::Env ort_env = Ort::Env(ORT_LOGGING_LEVEL_FATAL, "FastestDetEnd2");
    Ort::SessionOptions ort_session_opt = SessionOptions();
    Ort::MemoryInfo ort_memory_info = MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);
    Ort::AllocatorWithDefaultOptions ort_allocator;
    Ort::Session *ort_session = nullptr;

    // detect
    vector<char *> input_names;
    vector<char *> output_names;
    vector<int64_t> input_shape = {1, 3, 352, 352}; // n,c,h,w
    Size input_size = Size(352, 352);

    /**
     * @description: 维度变换
     * @param {Mat} &image_src
     * @return {void}
     */
    void hwc2chw2one(Mat &image_src)
    {
        vector<Mat> image_bgr;
        split(image_src, image_bgr);

        // Stretch one-channel images to vector
        Mat b = image_bgr[0].reshape(1, 1);
        Mat g = image_bgr[1].reshape(1, 1);
        Mat r = image_bgr[2].reshape(1, 1);

        Mat bgr[] = {b, g, r};

        // Concatenate three vectors to one
        hconcat(bgr, 3, image_src);
    }
};

object_num 368

[CsVeryLoveXieWenLi]

I try one file

cpp success

/*
 * @Author: CsVeryLoveXieWenLi
 * @Date: 2022-09-13 11:34:05
 * @LastEditors: CsVeryLoveXieWenLi
 * @LastEditTime: 2022-10-28 02:22:55
 * @Description:
 * QQ: 1172236399
 * Sign: 唯有寒冷之意，才能让人绝境逢生。
 * Copyright (c) 2022 by CsVeryLoveXieWenLi, All Rights Reserved.
 */

#include <onnxruntime_cxx_api.h>
#include <stdio.h>
#include <iostream>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/imgproc.hpp>

using namespace Ort;
using namespace cv;
using namespace std;

/**
 * @description:
 * @param {vector<float>} &input 推理输入
 * @param {int} max_result 每一位结果一共有多少可能
 * @param {vector<string>} &keys 标签
 * @return {string}
 */
static string argmax(vector<float> &input, int64_t max_result, vector<string> keys)
{
    int64_t max;
    int64_t old;
    vector<float>::iterator start;
    string result;
    for (size_t index = 0; index < input.size(); index += max_result)
    {
        start = input.begin() + index;
        max = (int64_t)(max_element(start, start + max_result) - start);
        if (max != old)
        {
            result.append(keys[max]);
        }
        old = max;
    }
    cout << "success: " << result.data() << endl;
    return result;
}

/**
 * @description: 维度变换
 * @param {Mat} &image_src
 * @return {void}
 */
void hwc2chw2one(Mat &image_src)
{
    vector<Mat> image_bgr;
    split(image_src, image_bgr);

    // Stretch one-channel images to vector
    Mat b = image_bgr[0].reshape(1, 1);
    Mat g = image_bgr[1].reshape(1, 1);
    Mat r = image_bgr[2].reshape(1, 1);

    Mat bgr[] = {b, g, r};

    // Concatenate three vectors to one
    hconcat(bgr, 3, image_src);
}

int main()
{

    // 初始化环境变量 -> 配置 -> 8线程推理 -> 启用所有优化 -> 加载模型 -> 定义标签
    Env env = Env(ORT_LOGGING_LEVEL_ERROR, "test");
    Ort::Session *session = nullptr;
    SessionOptions sessionOptions = SessionOptions();
    sessionOptions.SetIntraOpNumThreads(8);
    sessionOptions.SetGraphOptimizationLevel(ORT_ENABLE_ALL);
    session = new Session(env, "FastestDet2.onnx", sessionOptions);

    // 定义 输入和输出 名称
    vector<const char *> input_names = {"image"};
    cout << "input_name: " << input_names[0] << endl;
    vector<const char *> output_names = {"box"};
    cout << "output_name: " << output_names[0] << endl;

    // 定义 输入输出 大小
    vector<int64_t> input_shape = {1, 3, 352, 352};
    cout << "input_shape: " << input_shape.size() << endl;
    vector<int64_t> output_shape = {7, 1, 25};
    cout << "output_shape: " << output_shape.size() << endl;

    // 图片读取 -> 灰度化 -> float32 -> center -> hw to wh
    Mat image_src = imread("test.jpg");
    resize(image_src, image_src, Size(352, 352));
    // cvtColor(image_src, image_src, COLOR_BGR2GRAY);
    image_src.convertTo(image_src, CV_32FC3, 1.0 / 255);
    hwc2chw2one(image_src);
    // transpose(image_src, image_src);
    cout << "image_shape: " << image_src.cols << "," << image_src.rows << "," << image_src.total() << endl;

    // 申请内存环境 -> 初始化输入
    auto memory_info = MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);
    Value input_tensor = Value::CreateTensor<float>(memory_info, image_src.ptr<float>(), image_src.total(), input_shape.data(), input_shape.size());

    // 推理
    auto output = session->Run(Ort::RunOptions{nullptr}, input_names.data(), &input_tensor, input_names.size(), output_names.data(), output_names.size());
    float *floatarr = output.front().GetTensorMutableData<float>();
    vector<float> output_tensor(floatarr, floatarr + 175);

    cout << "output_shape: " << output[0].GetTensorTypeAndShapeInfo().GetShape()[0] << "," << output[0].GetTensorTypeAndShapeInfo().GetShape()[1] << endl;
    // argmax(output_tensor, 25, labels_keys);

    return 0;
}

input_name: image
output_name: box
input_shape: 4
output_shape: 3
image_shape: 371712,1,371712
output_shape: 3,6

[CsVeryLoveXieWenLi]

I'm sorry it's too late to see the normalization