Skip to content

Adapt easy deploy tool v2.0.0 #26

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zz990099 merged 2 commits into from
May 17, 2025
Merged

Adapt easy deploy tool v2.0.0 #26

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3f4d0cf
Update easy_deploy_tool submodule version
zz990099 May 17, 2025
fa86586
improve[FoundationPose]: Adapt EasyDeployTool v2.0.0
zz990099 May 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 47 additions & 59 deletions detection_6d_foundationpose/src/foundationpose.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,31 +124,18 @@ FoundationPose::FoundationPose(std::shared_ptr<inference_core::BaseInferCore>
{
// Check
auto refiner_blobs_buffer = refiner_core->GetBuffer(true);
if (refiner_blobs_buffer->GetOuterBlobBuffer(RENDER_INPUT_BLOB_NAME).first == nullptr)
auto scorer_blobs_buffer = scorer_core->GetBuffer(true);
try
{
LOG(ERROR) << "[FoundationPose] Failed to Construct FoundationPose since `renfiner_core` "
<< "do not has a blob named `" << RENDER_INPUT_BLOB_NAME << "`.";
throw std::runtime_error("[FoundationPose] Failed to Construct FoundationPose");
}
if (refiner_blobs_buffer->GetOuterBlobBuffer(TRANSF_INPUT_BLOB_NAME).first == nullptr)
{
LOG(ERROR) << "[FoundationPose] Failed to Construct FoundationPose since `renfiner_core` "
<< "do not has a blob named `" << TRANSF_INPUT_BLOB_NAME << "`.";
throw std::runtime_error("[FoundationPose] Failed to Construct FoundationPose");
}

auto scorer_blobs_buffer = scorer_core->GetBuffer(true);
if (scorer_blobs_buffer->GetOuterBlobBuffer(RENDER_INPUT_BLOB_NAME).first == nullptr)
{
LOG(ERROR) << "[FoundationPose] Failed to Construct FoundationPose since `scorer_core` "
<< "do not has a blob named `" << RENDER_INPUT_BLOB_NAME << "`.";
throw std::runtime_error("[FoundationPose] Failed to Construct FoundationPose");
}
if (scorer_blobs_buffer->GetOuterBlobBuffer(TRANSF_INPUT_BLOB_NAME).first == nullptr)
refiner_blobs_buffer->GetTensor(RENDER_INPUT_BLOB_NAME);
refiner_blobs_buffer->GetTensor(TRANSF_INPUT_BLOB_NAME);
scorer_blobs_buffer->GetTensor(RENDER_INPUT_BLOB_NAME);
scorer_blobs_buffer->GetTensor(TRANSF_INPUT_BLOB_NAME);
} catch (const std::exception &e)
{
LOG(ERROR) << "[FoundationPose] Failed to Construct FoundationPose since `scorer_core` "
<< "do not has a blob named `" << TRANSF_INPUT_BLOB_NAME << "`.";
throw std::runtime_error("[FoundationPose] Failed to Construct FoundationPose");
LOG(ERROR) << "[FoundationPose] Failed to Construct FoundationPose, ex : " << e.what();
throw std::runtime_error("[FoundationPose] Failed to Construct FoundationPose, ex : " +
std::string(e.what()));
}

// preload modules
Expand Down Expand Up @@ -210,7 +197,8 @@ bool FoundationPose::Register(const cv::Mat &rgb,
MESSURE_DURATION_AND_CHECK_STATE(UploadDataToDevice(rgb, depth, mask, package),
"[FoundationPose] SyncDetect Failed to upload data!!!");

for (size_t i = 0 ; i < refine_itr ; ++ i) {
for (size_t i = 0; i < refine_itr; ++i)
{
MESSURE_DURATION_AND_CHECK_STATE(
RefinePreProcess(package),
"[FoundationPose] SyncDetect Failed to execute RefinePreProcess!!!");
Expand Down Expand Up @@ -258,9 +246,10 @@ bool FoundationPose::Track(const cv::Mat &rgb,
MESSURE_DURATION_AND_CHECK_STATE(UploadDataToDevice(rgb, depth, cv::Mat(), package),
"[FoundationPose] Track Failed to upload data!!!");

for (size_t i = 0 ; i < refine_itr ; ++ i) {
MESSURE_DURATION_AND_CHECK_STATE(RefinePreProcess(package),
"[FoundationPose] Track Failed to execute RefinePreProcess!!!");
for (size_t i = 0; i < refine_itr; ++i)
{
MESSURE_DURATION_AND_CHECK_STATE(
RefinePreProcess(package), "[FoundationPose] Track Failed to execute RefinePreProcess!!!");

MESSURE_DURATION_AND_CHECK_STATE(
refiner_core_->SyncInfer(package->GetInferBuffer()),
Expand Down Expand Up @@ -337,44 +326,43 @@ bool FoundationPose::RefinePreProcess(const ParsingType &package)
}

// 2. render
if (package->refiner_blobs_buffer == nullptr) {
if (package->refiner_blobs_buffer == nullptr)
{
package->refiner_blobs_buffer = refiner_core_->GetBuffer(true);
}
const auto& refiner_blob_buffer = package->refiner_blobs_buffer;
const auto &refiner_blobs_buffer = package->refiner_blobs_buffer;
// 设置推理前blob的输入位置为device，输出的blob位置为host端
refiner_blob_buffer->SetBlobBuffer(RENDER_INPUT_BLOB_NAME, DataLocation::DEVICE);
refiner_blob_buffer->SetBlobBuffer(TRANSF_INPUT_BLOB_NAME, DataLocation::DEVICE);
refiner_blobs_buffer->GetTensor(RENDER_INPUT_BLOB_NAME)->SetBufferLocation(DataLocation::DEVICE);
refiner_blobs_buffer->GetTensor(TRANSF_INPUT_BLOB_NAME)->SetBufferLocation(DataLocation::DEVICE);

auto &refine_renderer = map_name2renderer_[package->target_name];
auto &refine_renderer = map_name2renderer_[package->target_name];
CHECK_STATE(
refine_renderer->RenderAndTransform(
package->hyp_poses, package->rgb_on_device.get(), package->depth_on_device.get(),
package->xyz_map_on_device.get(), package->input_image_height, package->input_image_width,
refiner_blob_buffer->GetOuterBlobBuffer(RENDER_INPUT_BLOB_NAME).first,
refiner_blob_buffer->GetOuterBlobBuffer(TRANSF_INPUT_BLOB_NAME).first,
refiner_blobs_buffer->GetTensor(RENDER_INPUT_BLOB_NAME)->RawPtr(),
refiner_blobs_buffer->GetTensor(TRANSF_INPUT_BLOB_NAME)->RawPtr(),
refine_mode_crop_ratio_),
"[FoundationPose] Failed to render and transform !!!");
// 3. 设置推理时形状
const int input_poses_num = package->hyp_poses.size();
refiner_blob_buffer->SetBlobShape(RENDER_INPUT_BLOB_NAME,
{input_poses_num, crop_window_H_, crop_window_W_, 6});
refiner_blob_buffer->SetBlobShape(TRANSF_INPUT_BLOB_NAME,
{input_poses_num, crop_window_H_, crop_window_W_, 6});
package->infer_buffer = refiner_blob_buffer;
const size_t input_poses_num = package->hyp_poses.size();
refiner_blobs_buffer->GetTensor(RENDER_INPUT_BLOB_NAME)
->SetShape({input_poses_num, static_cast<uint64_t>(crop_window_H_),
static_cast<uint64_t>(crop_window_W_), 6});
refiner_blobs_buffer->GetTensor(TRANSF_INPUT_BLOB_NAME)
->SetShape({input_poses_num, static_cast<uint64_t>(crop_window_H_),
static_cast<uint64_t>(crop_window_W_), 6});
package->infer_buffer = refiner_blobs_buffer.get();

return true;
}

bool FoundationPose::RefinePostProcess(const ParsingType &package)
{
// 获取refiner模型的缓存指针
const auto &refiner_blob_buffer = package->refiner_blobs_buffer;
const auto _trans_ptr = refiner_blob_buffer->GetOuterBlobBuffer(REFINE_TRANS_OUT_BLOB_NAME).first;
const auto _rot_ptr = refiner_blob_buffer->GetOuterBlobBuffer(REFINE_ROT_OUT_BLOB_NAME).first;
const float *trans_ptr = static_cast<float *>(_trans_ptr);
const float *rot_ptr = static_cast<float *>(_rot_ptr);
CHECK_STATE(trans_ptr != nullptr, "[FoundationPose] RefinePostProcess got invalid trans_ptr !");
CHECK_STATE(rot_ptr != nullptr, "[FoundationPose] RefinePostProcess got invalid rot_ptr !");
const auto &refiner_blobs_buffer = package->refiner_blobs_buffer;
const auto trans_ptr = refiner_blobs_buffer->GetTensor(REFINE_TRANS_OUT_BLOB_NAME)->Cast<float>();
const auto rot_ptr = refiner_blobs_buffer->GetTensor(REFINE_ROT_OUT_BLOB_NAME)->Cast<float>();

// 获取生成的假设位姿
const auto &hyp_poses = package->hyp_poses;
Expand Down Expand Up @@ -419,39 +407,39 @@ bool FoundationPose::RefinePostProcess(const ParsingType &package)

bool FoundationPose::ScorePreprocess(const ParsingType &package)
{
auto scorer_blob_buffer = scorer_core_->GetBuffer(false);
auto scorer_blobs_buffer = scorer_core_->GetBuffer(false);
// 获取对应的score_renderer
// 设置推理前后blob输出的位置，这里输入输出都在device端
scorer_blob_buffer->SetBlobBuffer(RENDER_INPUT_BLOB_NAME, DataLocation::DEVICE);
scorer_blob_buffer->SetBlobBuffer(TRANSF_INPUT_BLOB_NAME, DataLocation::DEVICE);
scorer_blob_buffer->SetBlobBuffer(SCORE_OUTPUT_BLOB_NAME, DataLocation::DEVICE);
scorer_blobs_buffer->GetTensor(RENDER_INPUT_BLOB_NAME)->SetBufferLocation(DataLocation::DEVICE);
scorer_blobs_buffer->GetTensor(TRANSF_INPUT_BLOB_NAME)->SetBufferLocation(DataLocation::DEVICE);
scorer_blobs_buffer->GetTensor(SCORE_OUTPUT_BLOB_NAME)->SetBufferLocation(DataLocation::DEVICE);

auto &score_renderer = map_name2renderer_[package->target_name];
CHECK_STATE(
score_renderer->RenderAndTransform(
package->hyp_poses, package->rgb_on_device.get(), package->depth_on_device.get(),
package->xyz_map_on_device.get(), package->input_image_height, package->input_image_width,
scorer_blob_buffer->GetOuterBlobBuffer(RENDER_INPUT_BLOB_NAME).first,
scorer_blob_buffer->GetOuterBlobBuffer(TRANSF_INPUT_BLOB_NAME).first,
score_mode_crop_ratio_),
scorer_blobs_buffer->GetTensor(RENDER_INPUT_BLOB_NAME)->RawPtr(),
scorer_blobs_buffer->GetTensor(TRANSF_INPUT_BLOB_NAME)->RawPtr(), score_mode_crop_ratio_),
"[FoundationPose] score_renderer RenderAndTransform Failed!!!");

package->scorer_blobs_buffer = scorer_blob_buffer;
package->infer_buffer = scorer_blob_buffer;
package->scorer_blobs_buffer = scorer_blobs_buffer;
package->infer_buffer = scorer_blobs_buffer.get();

return true;
}

bool FoundationPose::ScorePostProcess(const ParsingType &package)
{
const auto &scorer_blob_buffer = package->scorer_blobs_buffer;
const auto &scorer_blobs_buffer = package->scorer_blobs_buffer;
// 获取scorer模型的输出缓存指针
void *score_ptr = scorer_blob_buffer->GetOuterBlobBuffer(SCORE_OUTPUT_BLOB_NAME).first;
const auto score_ptr = scorer_blobs_buffer->GetTensor(SCORE_OUTPUT_BLOB_NAME)->Cast<float>();

const auto &refine_poses = package->hyp_poses;
const int poses_num = refine_poses.size();

// 获取置信度最大的refined_pose
int max_score_index = getMaxScoreIndex(nullptr, reinterpret_cast<float *>(score_ptr), poses_num);
int max_score_index = getMaxScoreIndex(nullptr, score_ptr, poses_num);
package->actual_pose = refine_poses[max_score_index];

return true;
Expand Down
8 changes: 4 additions & 4 deletions detection_6d_foundationpose/src/foundationpose_utils.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,16 +58,16 @@ struct FoundationPosePipelinePackage : public async_pipeline::IPipelinePackage {
std::vector<Eigen::Matrix4f> hyp_poses;

// 保存refine阶段用的推理缓存
std::shared_ptr<inference_core::IBlobsBuffer> refiner_blobs_buffer;
std::shared_ptr<inference_core::BlobsTensor> refiner_blobs_buffer;
// 保存score阶段用的推理缓存
std::shared_ptr<inference_core::IBlobsBuffer> scorer_blobs_buffer;
std::shared_ptr<inference_core::BlobsTensor> scorer_blobs_buffer;
// 保存用于推理的blob_buffer
std::shared_ptr<inference_core::IBlobsBuffer> infer_buffer;
inference_core::BlobsTensor* infer_buffer;

// **最终输出的位姿** //
Eigen::Matrix4f actual_pose;

std::shared_ptr<inference_core::IBlobsBuffer> GetInferBuffer()
inference_core::BlobsTensor* GetInferBuffer()
{
return infer_buffer;
}
Expand Down
2 changes: 1 addition & 1 deletion easy_deploy_tool
Open in desktop
Submodule easy_deploy_tool updated 57 files
+2 −2 .clang-format
+4 −4 .github/workflows/Compilation.yml
+15 −16 .github/workflows/ImageBuild.yml
+26 −0 .github/workflows/Lint.yml
+30 −0 .pre-commit-config.yaml
+3 −3 CMakeLists.txt
+16 −16 README.md
+2 −2 deploy_core/CMakeLists.txt
+13 −14 deploy_core/README.md
+5 −4 deploy_core/include/deploy_core/async_pipeline.h
+4 −3 deploy_core/include/deploy_core/async_pipeline_impl.h
+6 −7 deploy_core/include/deploy_core/base_detection.h
+25 −24 deploy_core/include/deploy_core/base_infer_core.h
+6 −7 deploy_core/include/deploy_core/base_sam.h
+14 −13 deploy_core/include/deploy_core/base_stereo.h
+76 −110 deploy_core/include/deploy_core/blob_buffer.h
+13 −9 deploy_core/include/deploy_core/block_queue.h
+10 −1 deploy_core/include/deploy_core/common_defination.h
+1 −1 deploy_core/include/deploy_core/wrapper.h
+7 −7 deploy_core/src/base_detection.cpp
+6 −6 deploy_core/src/base_infer_core.cpp
+2 −2 deploy_core/src/base_mono_stereo.cpp
+1 −1 deploy_core/src/base_sam.cpp
+2 −2 deploy_core/src/base_stereo.cpp
+5 −5 deploy_utils/image_processing_utils/CMakeLists.txt
+3 −2 deploy_utils/image_processing_utils/include/detection_2d_util/detection_2d_util.h
+1 −1 deploy_utils/image_processing_utils/src/detection_2d_cuda_preprocess.cu
+1 −1 deploy_utils/image_processing_utils/src/detection_2d_postprocess_cpu.cpp
+4 −9 deploy_utils/image_processing_utils/src/detection_2d_preprocess_cpu.cpp
+5 −9 deploy_utils/image_processing_utils/src/detection_2d_preprocess_cuda.cpp
+2 −2 doc/EnviromentSetup.md
+6 −6 doc/HowToDeployModels.md
+3 −3 docker/build_docker.sh
+8 −8 docker/easy_deploy_startup.sh
+1 −1 docker/jetson_tensorrt_trt8_u2004.dockerfile
+1 −1 docker/jetson_tensorrt_trt8_u2204.dockerfile
+5 −5 docker/nvidia_gpu_tensorrt_trt8_u2004.dockerfile
+1 −1 docker/nvidia_gpu_tensorrt_trt8_u2204.dockerfile
+2 −2 docker/rknn_230_u2204.dockerfile
+1 −1 inference_core/CMakeLists.txt
+4 −4 inference_core/ort_core/CMakeLists.txt
+9 −9 inference_core/ort_core/include/ort_core/ort_core.h
+146 −0 inference_core/ort_core/src/ort_blob_buffer.hpp
+116 −246 inference_core/ort_core/src/ort_core.cpp
+5 −5 inference_core/ort_core/src/ort_core_factory.cpp
+2 −2 inference_core/rknn_core/CMakeLists.txt
+1 −1 inference_core/rknn_core/include/rknn_core/rknn_core.h
+145 −0 inference_core/rknn_core/src/rknn_blob_buffer.hpp
+122 −288 inference_core/rknn_core/src/rknn_core.cpp
+1 −1 inference_core/rknn_core/src/rknn_core_factory.cpp
+3 −3 inference_core/trt_core/CMakeLists.txt
+8 −8 inference_core/trt_core/include/trt_core/trt_core.h
+112 −177 inference_core/trt_core/src/trt_blob_buffer.hpp
+106 −156 inference_core/trt_core/src/trt_core.cpp
+5 −5 inference_core/trt_core/src/trt_core_factory.cpp
+1 −1 tools/rk3588_echo_npu_usage.sh
+1 −1 tools/rk3588_max_performance.sh