Skip to content

Commit c70e1ff

Browse files
[ORT][TRT] support FaceFusion (#445)
1 parent 4252d27 commit c70e1ff

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

55 files changed

+3313
-34
lines changed

CMakeLists.txt

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,18 @@ if (CMAKE_COMPILER_IS_GNUCXX)
1919
set(CMAKE_CXX_FLAGS "-std=c++17 -Wno-deprecated ${CMAKE_CXX_FLAGS} ")
2020
message(STATUS "[Linux GCC Compiler Options]+:-std=c++17 -Wno-deprecated")
2121
endif ()
22+
# 指定CUDA编译器
23+
set(CMAKE_CUDA_COMPILER "/usr/local/cuda/bin/nvcc")
24+
set(CMAKE_CUDA_ARCHITECTURES 89) # For RTX 20xx series
2225

26+
# 启用 CUDA
27+
enable_language(CUDA)
2328
set(LITE_AI_ROOT_DIR ${CMAKE_SOURCE_DIR})
2429

25-
option(ENABLE_TEST "build test examples." OFF)
30+
option(ENABLE_TEST "build test examples." ON)
2631
option(ENABLE_DEBUG_STRING "enable DEBUG string or not" ON)
2732
option(ENABLE_ONNXRUNTIME "enable ONNXRuntime engine" ON)
28-
option(ENABLE_TENSORRT "enable TensorRT engine" OFF)
33+
option(ENABLE_TENSORRT "enable TensorRT engine" ON)
2934
option(ENABLE_MNN "enable MNN engine" OFF)
3035
option(ENABLE_NCNN "enable NCNN engine" OFF)
3136
option(ENABLE_TNN "enable TNN engine" OFF)

cmake/opencv.cmake

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@ link_directories(${OpenCV_DIR}/lib)
1919

2020
if(NOT WIN32)
2121
if(ENABLE_OPENCV_VIDEOIO OR ENABLE_TEST)
22-
set(OpenCV_LIBS opencv_core opencv_imgproc opencv_imgcodecs opencv_video opencv_videoio)
22+
set(OpenCV_LIBS opencv_core opencv_imgproc opencv_imgcodecs opencv_video opencv_videoio opencv_calib3d)
2323
else()
24-
set(OpenCV_LIBS opencv_core opencv_imgproc opencv_imgcodecs) # no videoio, video module
24+
set(OpenCV_LIBS opencv_core opencv_imgproc opencv_imgcodecs opencv_calib3d) # no videoio, video module
2525
endif()
2626
else()
2727
set(OpenCV_LIBS opencv_world490)

cmake/tensorrt.cmake

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ link_directories(${TensorRT_DIR}/lib)
4141

4242
# 1. glob sources files
4343
file(GLOB TENSORRT_CORE_SRCS ${CMAKE_SOURCE_DIR}/lite/trt/core/*.cpp)
44+
file(GLOB TENSORRT_CUDA_KERNEL_SRCS_CPP ${CMAKE_SOURCE_DIR}/lite/trt/kernel/*.cpp)
45+
file(GLOB TENSORRT_CUDA_KERNEL_SRCS_CU ${CMAKE_SOURCE_DIR}/lite/trt/kernel/*.cu)
4446
file(GLOB TENSORRT_CV_SRCS ${CMAKE_SOURCE_DIR}/lite/trt/cv/*.cpp)
4547
file(GLOB TENSORRT_NLP_SRCS ${CMAKE_SOURCE_DIR}/lite/trt/nlp/*.cpp)
4648
file(GLOB TENSORRT_ASR_SRCS ${CMAKE_SOURCE_DIR}/lite/trt/asr/*.cpp)
@@ -52,8 +54,17 @@ file(GLOB TENSORRT_CV_HEAD ${CMAKE_SOURCE_DIR}/lite/trt/cv/*.h)
5254
file(GLOB TENSORRT_NLP_HEAD ${CMAKE_SOURCE_DIR}/lite/trt/nlp/*.h)
5355
file(GLOB TENSORRT_ASR_HEAD ${CMAKE_SOURCE_DIR}/lite/trt/asr/*.h)
5456
file(GLOB TENSORRT_SD_HEAD ${CMAKE_SOURCE_DIR}/lite/trt/sd/*.h)
57+
file(GLOB TENSORRT_CUDA_KERNEL_HEAD_CPP ${CMAKE_SOURCE_DIR}/lite/trt/kernel/*.h)
58+
file(GLOB TENSORRT_CUDA_KERNEL_HEAD_CU ${CMAKE_SOURCE_DIR}/lite/trt/kernel/*.cuh)
59+
60+
61+
62+
set(TRT_SRCS ${TENSORRT_CV_SRCS} ${TENSORRT_NLP_SRCS} ${TENSORRT_ASR_SRCS} ${TENSORRT_CORE_SRCS} ${TENSORRT_SD_SRCS}
63+
${TENSORRT_CUDA_KERNEL_SRCS_CPP} ${TENSORRT_CUDA_KERNEL_SRCS_CU})
64+
set_source_files_properties(${TENSORRT_CUDA_KERNEL_SRCS_CU} ${TENSORRT_CUDA_KERNEL_SRCS_CPP}
65+
${TENSORRT_CUDA_KERNEL_HEAD_CPP} ${TENSORRT_CUDA_KERNEL_HEAD_CU}
66+
PROPERTIES LANGUAGE CUDA)
5567

56-
set(TRT_SRCS ${TENSORRT_CV_SRCS} ${TENSORRT_NLP_SRCS} ${TENSORRT_ASR_SRCS} ${TENSORRT_CORE_SRCS} ${TENSORRT_SD_SRCS})
5768
# 3. copy
5869
message("[Lite.AI.Toolkit][I] Installing Lite.AI.ToolKit Headers for TensorRT Backend ...")
5970
# "INSTALL" can copy all files from the list to the specified path.
@@ -63,4 +74,5 @@ file(INSTALL ${TENSORRT_CV_HEAD} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/lit
6374
file(INSTALL ${TENSORRT_ASR_HEAD} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/lite/trt/asr)
6475
file(INSTALL ${TENSORRT_NLP_HEAD} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/lite/trt/nlp)
6576
file(INSTALL ${TENSORRT_SD_HEAD} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/lite/trt/sd)
66-
77+
file(INSTALL ${TENSORRT_CUDA_KERNEL_HEAD_CPP} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/lite/trt/kernel)
78+
file(INSTALL ${TENSORRT_CUDA_KERNEL_HEAD_CU} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/lite/trt/kernel)

cmake/utils.cmake

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ function(add_lite_ai_toolkit_shared_library version soversion)
5252
if (ENABLE_TENSORRT)
5353
include(cmake/tensorrt.cmake)
5454
set(LITE_SRCS ${LITE_SRCS} ${TRT_SRCS})
55-
set(LITE_DEPENDENCIES ${LITE_DEPENDENCIES} cudart nvinfer nvonnxparser
55+
set(LITE_DEPENDENCIES ${LITE_DEPENDENCIES} cuda cudart nvinfer nvonnxparser
5656
nvinfer_plugin ddim_scheduler_cpp)
5757
link_directories(${CMAKE_SOURCE_DIR}/lite/bin)
5858
endif ()

examples/lite/CMakeLists.txt

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,5 +103,10 @@ add_lite_executable(lite_face_parsing_bisenet_dyn cv)
103103
add_lite_executable(lite_yolov8face cv)
104104
add_lite_executable(lite_lightenhance cv)
105105
add_lite_executable(lite_realesrgan cv)
106+
add_lite_executable(lite_face_68landmarks cv)
107+
add_lite_executable(lite_face_recognizer cv)
108+
add_lite_executable(lite_face_swap cv)
109+
add_lite_executable(lite_face_restoration cv)
110+
add_lite_executable(lite_facefusion_pipeline cv)
106111
add_lite_executable(lite_yolov8 cv)
107112
add_lite_executable(lite_sd_pipeline sd)
Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
//
2+
// Created by wangzijian on 11/1/24.
3+
//
4+
#include "lite/lite.h"
5+
#include "lite/trt/cv/trt_face_68landmarks_mt.h"
6+
7+
static void test_default()
8+
{
9+
#ifdef ENABLE_ONNXRUNTIME
10+
std::string onnx_path = "/home/lite.ai.toolkit/examples/hub/onnx/cv/2dfan4.onnx";
11+
std::string test_img_path = "/home/lite.ai.toolkit/examples/lite/resources/test_lite_facefusion_pipeline_source.jpg";
12+
13+
// 1. Test Default Engine ONNXRuntime
14+
lite::cv::faceid::Face_68Landmarks *face68Landmarks = new lite::cv::faceid::Face_68Landmarks(onnx_path);
15+
16+
lite::types::BoundingBoxType<float, float> bbox;
17+
bbox.x1 = 487;
18+
bbox.y1 = 236;
19+
bbox.x2 = 784;
20+
bbox.y2 = 624;
21+
22+
cv::Mat img_bgr = cv::imread(test_img_path);
23+
std::vector<cv::Point2f> face_landmark_5of68;
24+
face68Landmarks->detect(img_bgr, bbox, face_landmark_5of68);
25+
26+
std::cout<<"face id detect done!"<<std::endl;
27+
28+
delete face68Landmarks;
29+
#endif
30+
}
31+
32+
33+
34+
35+
static void test_tensorrt()
36+
{
37+
#ifdef ENABLE_TENSORRT
38+
std::string engine_path = "/home/lite.ai.toolkit/examples/hub/trt/2dfan4_fp16.engine";
39+
std::string test_img_path = "/home/lite.ai.toolkit/1.jpg";
40+
41+
// 1. Test TensorRT Engine
42+
lite::trt::cv::faceid::FaceFusionFace68Landmarks *face68Landmarks = new lite::trt::cv::faceid::FaceFusionFace68Landmarks(engine_path);
43+
lite::types::BoundingBoxType<float, float> bbox;
44+
bbox.x1 = 487;
45+
bbox.y1 = 236;
46+
bbox.x2 = 784;
47+
bbox.y2 = 624;
48+
49+
cv::Mat img_bgr = cv::imread(test_img_path);
50+
std::vector<cv::Point2f> face_landmark_5of68;
51+
face68Landmarks->detect(img_bgr, bbox, face_landmark_5of68);
52+
53+
std::cout<<"face id detect done!"<<std::endl;
54+
55+
delete face68Landmarks;
56+
#endif
57+
}
58+
59+
60+
static void test_tensorrt_mt()
61+
{
62+
#ifdef ENABLE_TENSORRT
63+
std::string engine_path = "/home/lite.ai.toolkit/examples/hub/trt/2dfan4_fp16.engine";
64+
std::string test_img_path = "/home/lite.ai.toolkit/1.jpg";
65+
66+
// 1. Test TensorRT Engine
67+
// lite::trt::cv::faceid::FaceFusionFace68Landmarks *face68Landmarks = new lite::trt::cv::faceid::FaceFusionFace68Landmarks(engine_path);
68+
trt_face_68landmarks_mt *face68Landmarks = new trt_face_68landmarks_mt(engine_path,4);
69+
70+
lite::types::BoundingBoxType<float, float> bbox;
71+
72+
bbox.x1 = 487;
73+
bbox.y1 = 236;
74+
bbox.x2 = 784;
75+
bbox.y2 = 624;
76+
77+
cv::Mat img_bgr = cv::imread(test_img_path);
78+
std::vector<cv::Point2f> face_landmark_5of68;
79+
face68Landmarks->detect_async(img_bgr, bbox, face_landmark_5of68);
80+
81+
cv::Mat img_bgr2 = cv::imread(test_img_path);
82+
std::vector<cv::Point2f> face_landmark_5of682;
83+
face68Landmarks->detect_async(img_bgr, bbox, face_landmark_5of682);
84+
85+
cv::Mat img_bgr3 = cv::imread(test_img_path);
86+
std::vector<cv::Point2f> face_landmark_5of683;
87+
face68Landmarks->detect_async(img_bgr, bbox, face_landmark_5of683);
88+
89+
90+
cv::Mat img_bgr4 = cv::imread(test_img_path);
91+
std::vector<cv::Point2f> face_landmark_5of684;
92+
face68Landmarks->detect_async(img_bgr, bbox, face_landmark_5of684);
93+
94+
face68Landmarks->wait_for_completion();
95+
96+
face68Landmarks->shutdown();
97+
98+
std::cout<<"face id detect done!"<<std::endl;
99+
100+
delete face68Landmarks;
101+
#endif
102+
}
103+
104+
105+
106+
int main(__unused int argc, __unused char *argv[])
107+
{
108+
// test_tensorrt();
109+
test_tensorrt_mt();
110+
// test_default();
111+
return 0;
112+
}
Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
//
2+
// Created by wangzijian on 11/5/24.
3+
//
4+
#include "lite/lite.h"
5+
6+
static void test_default()
7+
{
8+
#ifdef ENABLE_ONNXRUNTIME
9+
std::string onnx_path = "../../../examples/hub/onnx/cv/arcface_w600k_r50.onnx";
10+
std::string test_img_path = "../../../examples/lite/resources/test_lite_facefusion_pipeline_source.jpg";
11+
12+
// 1. Test Default Engine ONNXRuntime
13+
lite::cv::faceid::Face_Recognizer *face_recognizer = new lite::cv::faceid::Face_Recognizer(onnx_path);
14+
15+
std::vector<cv::Point2f> face_landmark_5 = {
16+
cv::Point2f(568.2485f, 398.9512f),
17+
cv::Point2f(701.7346f, 399.64795f),
18+
cv::Point2f(634.2213f, 482.92694f),
19+
cv::Point2f(583.5656f, 543.10187f),
20+
cv::Point2f(684.52405f, 543.125f)
21+
};
22+
cv::Mat img_bgr = cv::imread(test_img_path);
23+
24+
std::vector<float> source_image_embeding;
25+
26+
face_recognizer->detect(img_bgr,face_landmark_5,source_image_embeding);
27+
28+
29+
std::cout<<"face id detect done!"<<std::endl;
30+
31+
delete face_recognizer;
32+
#endif
33+
}
34+
35+
int main(__unused int argc, __unused char *argv[])
36+
{
37+
test_default();
38+
return 0;
39+
}
Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
//
2+
// Created by wangzijian on 11/7/24.
3+
//
4+
#include "lite/lite.h"
5+
6+
#include "lite/trt/cv/trt_face_restoration_mt.h"
7+
8+
static void test_default()
9+
{
10+
#ifdef ENABLE_ONNXRUNTIME
11+
std::string onnx_path = "/home/lite.ai.toolkit/examples/hub/onnx/cv/gfpgan_1.4.onnx";
12+
std::string test_img_path = "/home/lite.ai.toolkit/trt_result.jpg";
13+
std::string save_img_path = "/home/lite.ai.toolkit/trt_result_final.jpg";
14+
15+
// 1. Test Default Engine ONNXRuntime
16+
lite::cv::face::restoration::GFPGAN *face_restoration = new lite::cv::face::restoration::GFPGAN(onnx_path);
17+
18+
std::vector<cv::Point2f> face_landmark_5 = {
19+
cv::Point2f(569.092041f, 398.845886f),
20+
cv::Point2f(701.891724f, 399.156677f),
21+
cv::Point2f(634.767212f, 482.927216f),
22+
cv::Point2f(584.270996f, 543.294617f),
23+
cv::Point2f(684.877991f, 543.067078f)
24+
};
25+
cv::Mat img_bgr = cv::imread(test_img_path);
26+
27+
face_restoration->detect(img_bgr,face_landmark_5,save_img_path);
28+
29+
30+
std::cout<<"face restoration detect done!"<<std::endl;
31+
32+
delete face_restoration;
33+
#endif
34+
}
35+
36+
37+
38+
39+
static void test_tensorrt()
40+
{
41+
#ifdef ENABLE_TENSORRT
42+
std::string engine_path = "/home/lite.ai.toolkit/examples/hub/trt/gfpgan_1.4_fp32.engine";
43+
std::string test_img_path = "/home/lite.ai.toolkit/trt_result.jpg";
44+
std::string save_img_path = "/home/lite.ai.toolkit/trt_facerestoration_mt_test111.jpg";
45+
46+
// 1. Test Default Engine TensorRT
47+
// lite::trt::cv::face::restoration::TRTGFPGAN *face_restoration_trt = new lite::trt::cv::face::restoration::TRTGFPGAN(engine_path);
48+
49+
const int num_threads = 4; // 使用4个线程
50+
auto face_restoration_trt = std::make_unique<trt_face_restoration_mt>(engine_path,4);
51+
52+
// trt_face_restoration_mt *face_restoration_trt = new trt_face_restoration_mt(engine_path);
53+
54+
55+
// 2. 准备测试数据 - 这里假设我们要处理4张相同的图片作为示例
56+
std::vector<std::string> test_img_paths = {
57+
"/home/lite.ai.toolkit/trt_result.jpg",
58+
"/home/lite.ai.toolkit/trt_result_2.jpg",
59+
"/home/lite.ai.toolkit/trt_result_3.jpg",
60+
"/home/lite.ai.toolkit/trt_result_4.jpg"
61+
};
62+
63+
std::vector<std::string> save_img_paths = {
64+
"/home/lite.ai.toolkit/trt_facerestoration_mt_thread1.jpg",
65+
"/home/lite.ai.toolkit/trt_facerestoration_mt_thread2.jpg",
66+
"/home/lite.ai.toolkit/trt_facerestoration_mt_thread3.jpg",
67+
"/home/lite.ai.toolkit/trt_facerestoration_mt_thread4.jpg"
68+
};
69+
70+
std::vector<cv::Point2f> face_landmark_5 = {
71+
cv::Point2f(569.092041f, 398.845886f),
72+
cv::Point2f(701.891724f, 399.156677f),
73+
cv::Point2f(634.767212f, 482.927216f),
74+
cv::Point2f(584.270996f, 543.294617f),
75+
cv::Point2f(684.877991f, 543.067078f)
76+
};
77+
// cv::Mat img_bgr = cv::imread(test_img_path);
78+
//
79+
// face_restoration_trt->detect_async(img_bgr,face_landmark_5,save_img_path);
80+
//
81+
//
82+
// std::cout<<"face restoration detect done!"<<std::endl;
83+
//
84+
// delete face_restoration_trt;
85+
auto start_time = std::chrono::high_resolution_clock::now();
86+
87+
for (size_t i=0; i < test_img_paths.size();++i){
88+
cv::Mat img_bgr = cv::imread(test_img_paths[i]);
89+
if (img_bgr.empty()) {
90+
std::cerr << "Failed to read image: " << test_img_paths[i] << std::endl;
91+
continue;
92+
}
93+
// 异步提交任务
94+
face_restoration_trt->detect_async(img_bgr, face_landmark_5, save_img_paths[i]);
95+
std::cout << "Submitted task " << i + 1 << " for processing" << std::endl;
96+
}
97+
98+
// 6. 等待所有任务完成
99+
std::cout << "Waiting for all tasks to complete..." << std::endl;
100+
face_restoration_trt->wait_for_completion();
101+
102+
// 7. 计算和输出总耗时
103+
auto end_time = std::chrono::high_resolution_clock::now();
104+
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time);
105+
106+
std::cout << "All tasks completed!" << std::endl;
107+
std::cout << "Total processing time: " << duration.count() << "ms" << std::endl;
108+
std::cout << "Average time per image: " << duration.count() / test_img_paths.size() << "ms" << std::endl;
109+
110+
111+
#endif
112+
}
113+
114+
int main(__unused int argc, __unused char *argv[])
115+
{
116+
// test_default();
117+
test_tensorrt();
118+
return 0;
119+
}

0 commit comments

Comments
 (0)