Inference with trt plugin in C++.

[SirojbekSafarov]

Hello, Thank you for the great work.

My question might be a little off-topic.

I am trying to do inference trt model with custom plugins. I get custom plugin source code from your source code. When I do inference everything is working fine but the results are wrong. I also do inference with another trt model (which does not include trt plugin) it is working well and the performance also good. I also convert trt plugins to .dll file and do inference with python it is also working. I used trtexec with loading trt plugin it is also working and the performance is good. I just want to know do we need to do any extra steps if we want to use trt plugins in c++.

bool EngineLoader::runInference(const std::vectorcv::Mat& inputBatchImages, std::vector<std::vector>& featureVectors) {
auto dims = m_engine->getBindingDimensions(0);

CHECK(cudaStreamCreate(&m_cudaStream));

Dims4 inputDims = { static_cast<int32_t>(inputBatchImages.size()), dims.d[1], dims.d[2], dims.d[3] };
m_context->setBindingDimensions(0, inputDims);

if (!m_context->allInputDimensionsSpecified()) {
	throw std::runtime_error("Error, not all input dimensions specified.");
}

auto batchSize = static_cast<int32_t>(inputBatchImages.size());

// Only reallocate buffers if the batch size has changed
if (m_prevBatchSize != inputBatchImages.size()) {

	m_inputBuff.hostBuffer.resize(inputDims);
	m_inputBuff.deviceBuffer.resize(inputDims);

	Dims3 outputDims{batchSize, 2000,  37};
	m_outputBuff.hostBuffer.resize(outputDims);
	m_outputBuff.deviceBuffer.resize(outputDims);

	m_prevBatchSize = batchSize;
}

auto* hostDataBuffer = static_cast<float*>(m_inputBuff.hostBuffer.data());

for (size_t batch = 0; batch < inputBatchImages.size(); ++batch) {
	auto image = inputBatchImages[batch];

	const float subR = 123.675;
	const float divR = 58.395;
	const float subG = 116.28;
	const float divG = 57.12;
	const float subB = 103.53;
	const float divB = 57.375;

	int offset = dims.d[1] * dims.d[2] * dims.d[3] * batch;
	int r = 0, g = 0, b = 0;
	for (int i = 0; i < dims.d[1] * dims.d[2] * dims.d[3]; ++i) {
		if (i % 3 == 0) {
			hostDataBuffer[offset + r++] = (float(*(reinterpret_cast<unsigned char*>(image.data) + i)) - subR)/ divR;
		}
		else if (i % 3 == 1) {
			hostDataBuffer[offset + g++ + dims.d[2] * dims.d[3]] = (float(*(reinterpret_cast<unsigned char*>(image.data) + i)) - subG) / divG;
		}
		else {
			hostDataBuffer[offset + b++ + dims.d[2] * dims.d[3] * 2] = (float(*(reinterpret_cast<unsigned char*>(image.data) + i)) - subB) / divB;
		}
	}
}

// Copy from CPU to GPU
auto ret = cudaMemcpyAsync(m_inputBuff.deviceBuffer.data(), m_inputBuff.hostBuffer.data(), m_inputBuff.hostBuffer.nbBytes(), cudaMemcpyHostToDevice, m_cudaStream);

std::vector<void*> predicitonBindings = { m_inputBuff.deviceBuffer.data(), m_outputBuff.deviceBuffer.data() };

// Run inference.
bool status = m_context->enqueueV2(predicitonBindings.data(), m_cudaStream, nullptr);

// Copy the results back to CPU memory
ret = cudaMemcpyAsync(m_outputBuff.hostBuffer.data(), m_outputBuff.deviceBuffer.data(), m_outputBuff.deviceBuffer.nbBytes(), cudaMemcpyDeviceToHost, m_cudaStream);

ret = cudaStreamSynchronize(m_cudaStream);

}

This is my code. If you have any suggestions please give me. I am just stuck here I do not know what to do.
Thank you!

[lvhan028]

What's the pixel format of input mats? If it is BGR, I think branches if (i % 3 == 0) and else should be switched.
Also, how did you integrate mmdeploy custom tensorrt codes to your project?