Skip to content

Commit a14177c

Browse files
irexycRunningLeon
irexyc
and
RunningLeon
authored
[Feature]: Add github prebuild workflow after new release. (#1852)
* add prebuild dockerfile * add prebuild test workflw * update * update * rm other workflow for test * Update docker image * add win1o prebuild * add test prebuild * add windows scripts in prebuilt package * add linux scripts in prebuilt package * generate_build_config.py * fix cudnn search * fix env * fix script * fix rpath * fix cwd * fix windows * fix lint * windows prebuild ci * linux prebuild ci * fix * update trigger * Revert "rm other workflow for test" This reverts commit 0a03872. * update sdk build readme * update prebuild * fix dll deps for python >= 3.8 on windows * fix ci * test prebuild * update test script to avoid modify upload folder * add onnxruntime.dll to mmdeploy_python * update prebuild workflow * update prebuild * Update loader.cpp.in * remove exists prebuild files * fix opencv env * update cmake options for mmdeploy python build * remove test code * fix lint --------- Co-authored-by: RunningLeon <[email protected]> Co-authored-by: RunningLeon <[email protected]>
1 parent 9f9b3a8 commit a14177c

File tree

28 files changed

+1313
-215
lines changed

28 files changed

+1313
-215
lines changed

‎.github/workflows/prebuild.yml‎

Lines changed: 233 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,233 @@
1+
name: prebuild
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
- dev-1.x
8+
paths:
9+
- "mmdeploy/version.py"
10+
11+
permissions: write-all
12+
13+
jobs:
14+
linux_build:
15+
runs-on: [self-hosted, linux-3090]
16+
container:
17+
image: openmmlab/mmdeploy:manylinux2014_x86_64-cuda11.3
18+
options: "--gpus=all --ipc=host"
19+
volumes:
20+
- /data2/actions-runner/prebuild:/__w/mmdeploy/prebuild
21+
steps:
22+
- name: Checkout repository
23+
uses: actions/checkout@v3
24+
with:
25+
submodules: recursive
26+
- name: Get mmdeploy version
27+
run: |
28+
export MMDEPLOY_VERSION=$(python3 -c "import sys; sys.path.append('mmdeploy');from version import __version__;print(__version__)")
29+
echo $MMDEPLOY_VERSION
30+
echo "MMDEPLOY_VERSION=$MMDEPLOY_VERSION" >> $GITHUB_ENV
31+
- name: Build MMDeploy
32+
run: |
33+
source activate mmdeploy-3.6
34+
pip install pyyaml packaging setuptools wheel
35+
mkdir pack; cd pack
36+
python ../tools/package_tools/generate_build_config.py --backend 'trt;ort' \
37+
--system linux --output config.yml --build-mmdeploy
38+
python ../tools/package_tools/mmdeploy_builder.py --config config.yml
39+
- name: Build sdk cpu backend
40+
run: |
41+
source activate mmdeploy-3.6
42+
cd pack
43+
python ../tools/package_tools/generate_build_config.py --backend 'ort' \
44+
--system linux --output config.yml --device cpu --build-sdk --build-sdk-monolithic \
45+
--build-sdk-python --sdk-dynamic-net
46+
python ../tools/package_tools/mmdeploy_builder.py --config config.yml
47+
- name: Build sdk cuda backend
48+
run: |
49+
source activate mmdeploy-3.6
50+
cd pack
51+
python ../tools/package_tools/generate_build_config.py --backend 'ort;trt' \
52+
--system linux --output config.yml --device cuda --build-sdk --build-sdk-monolithic \
53+
--build-sdk-python --sdk-dynamic-net
54+
python ../tools/package_tools/mmdeploy_builder.py --config config.yml
55+
- name: Move artifact
56+
run: |
57+
mkdir -p /__w/mmdeploy/prebuild/$MMDEPLOY_VERSION
58+
rm -rf /__w/mmdeploy/prebuild/$MMDEPLOY_VERSION/*
59+
mv pack/* /__w/mmdeploy/prebuild/$MMDEPLOY_VERSION
60+
61+
linux_test:
62+
runs-on: [self-hosted, linux-3090]
63+
needs: linux_build
64+
container:
65+
image: openmmlab/mmdeploy:ubuntu20.04-cuda11.3
66+
options: "--gpus=all --ipc=host"
67+
volumes:
68+
- /data2/actions-runner/prebuild:/__w/mmdeploy/prebuild
69+
- /data2/actions-runner/testmodel:/__w/mmdeploy/testmodel
70+
steps:
71+
- name: Checkout repository
72+
uses: actions/checkout@v3
73+
- name: Get mmdeploy version
74+
run: |
75+
export MMDEPLOY_VERSION=$(python3 -c "import sys; sys.path.append('mmdeploy');from version import __version__;print(__version__)")
76+
echo $MMDEPLOY_VERSION
77+
echo "MMDEPLOY_VERSION=$MMDEPLOY_VERSION" >> $GITHUB_ENV
78+
- name: Test python
79+
run: |
80+
cd /__w/mmdeploy/prebuild/$MMDEPLOY_VERSION
81+
bash $GITHUB_WORKSPACE/tools/package_tools/test/test_sdk_python.sh
82+
- name: Test c/cpp
83+
run: |
84+
cd /__w/mmdeploy/prebuild/$MMDEPLOY_VERSION
85+
bash $GITHUB_WORKSPACE/tools/package_tools/test/test_sdk.sh
86+
87+
linux_upload:
88+
runs-on: [self-hosted, linux-3090]
89+
if: startsWith(github.ref, 'refs/tags/')
90+
environment: 'prod'
91+
needs: linux_test
92+
env:
93+
PREBUILD_DIR: /data2/actions-runner/prebuild
94+
steps:
95+
- name: Checkout repository
96+
uses: actions/checkout@v3
97+
- name: Get mmdeploy version
98+
run: |
99+
export MMDEPLOY_VERSION=$(python3 -c "import sys; sys.path.append('mmdeploy');from version import __version__;print(__version__)")
100+
echo $MMDEPLOY_VERSION
101+
echo "MMDEPLOY_VERSION=$MMDEPLOY_VERSION" >> $GITHUB_ENV
102+
- name: Upload mmdeploy
103+
run: |
104+
cd $PREBUILD_DIR/$MMDEPLOY_VERSION/mmdeploy
105+
pip install twine
106+
# twine upload * --repository testpypi -u __token__ -p ${{ secrets.test_pypi_password }}
107+
twine upload * -u __token__ -p ${{ secrets.pypi_password }}
108+
- name: Upload mmdeploy_python
109+
run: |
110+
cd $PREBUILD_DIR/$MMDEPLOY_VERSION/mmdeploy_python
111+
# twine upload * --repository testpypi -u __token__ -p ${{ secrets.test_pypi_password }}
112+
twine upload * -u __token__ -p ${{ secrets.pypi_password }}
113+
- name: Zip mmdeploy sdk
114+
run: |
115+
cd $PREBUILD_DIR/$MMDEPLOY_VERSION/sdk
116+
for folder in *
117+
do
118+
tar czf $folder.tar.gz $folder
119+
done
120+
- name: Upload mmdeploy sdk
121+
uses: softprops/action-gh-release@v1
122+
with:
123+
files: |
124+
$PREBUILD_DIR/$MMDEPLOY_VERSION/sdk/*.tar.gz
125+
126+
127+
windows_build:
128+
runs-on: [self-hosted, win10-3080]
129+
steps:
130+
- name: Checkout repository
131+
uses: actions/checkout@v3
132+
with:
133+
submodules: recursive
134+
- name: Get mmdeploy version
135+
run: |
136+
conda activate mmdeploy-3.8
137+
$env:MMDEPLOY_VERSION=(python -c "import sys; sys.path.append('mmdeploy');from version import __version__;print(__version__)")
138+
echo $env:MMDEPLOY_VERSION
139+
echo "MMDEPLOY_VERSION=$env:MMDEPLOY_VERSION" >> $env:GITHUB_ENV
140+
- name: Build MMDeploy
141+
run: |
142+
. D:\DEPS\cienv\prebuild_gpu_env.ps1
143+
conda activate mmdeploy-3.6
144+
mkdir pack; cd pack
145+
python ../tools/package_tools/generate_build_config.py --backend 'trt;ort' `
146+
--system windows --output config.yml --build-mmdeploy
147+
python ../tools/package_tools/mmdeploy_builder.py --config config.yml
148+
- name: Build sdk cpu backend
149+
run: |
150+
. D:\DEPS\cienv\prebuild_cpu_env.ps1
151+
conda activate mmdeploy-3.6
152+
cd pack
153+
python ../tools/package_tools/generate_build_config.py --backend 'ort' `
154+
--system windows --output config.yml --device cpu --build-sdk --build-sdk-monolithic `
155+
--build-sdk-python --sdk-dynamic-net
156+
python ../tools/package_tools/mmdeploy_builder.py --config config.yml
157+
- name: Build sdk cuda backend
158+
run: |
159+
. D:\DEPS\cienv\prebuild_gpu_env.ps1
160+
conda activate mmdeploy-3.6
161+
cd pack
162+
python ../tools/package_tools/generate_build_config.py --backend 'ort;trt' `
163+
--system windows --output config.yml --device cuda --build-sdk --build-sdk-monolithic `
164+
--build-sdk-python --sdk-dynamic-net
165+
python ../tools/package_tools/mmdeploy_builder.py --config config.yml
166+
- name: Move artifact
167+
run: |
168+
New-Item "D:/DEPS/ciartifact/$env:MMDEPLOY_VERSION" -ItemType Directory -Force
169+
Remove-Item "D:/DEPS/ciartifact/$env:MMDEPLOY_VERSION/*" -Force -Recurse
170+
Move-Item pack/* "D:/DEPS/ciartifact/$env:MMDEPLOY_VERSION"
171+
172+
windows_test:
173+
runs-on: [self-hosted, win10-3080]
174+
needs: windows_build
175+
steps:
176+
- name: Checkout repository
177+
uses: actions/checkout@v3
178+
- name: Get mmdeploy version
179+
run: |
180+
conda activate mmdeploy-3.8
181+
$env:MMDEPLOY_VERSION=(python -c "import sys; sys.path.append('mmdeploy');from version import __version__;print(__version__)")
182+
echo $env:MMDEPLOY_VERSION
183+
echo "MMDEPLOY_VERSION=$env:MMDEPLOY_VERSION" >> $env:GITHUB_ENV
184+
- name: Test python
185+
run: |
186+
cd "D:/DEPS/ciartifact/$env:MMDEPLOY_VERSION"
187+
. D:\DEPS\cienv\prebuild_cpu_env.ps1
188+
conda activate ci-test
189+
& "$env:GITHUB_WORKSPACE/tools/package_tools/test/test_sdk_python.ps1"
190+
- name: Test c/cpp
191+
run: |
192+
cd "D:/DEPS/ciartifact/$env:MMDEPLOY_VERSION"
193+
. D:\DEPS\cienv\prebuild_cpu_env.ps1
194+
& "$env:GITHUB_WORKSPACE/tools/package_tools/test/test_sdk.ps1"
195+
196+
windows_upload:
197+
runs-on: [self-hosted, win10-3080]
198+
if: startsWith(github.ref, 'refs/tags/')
199+
environment: 'prod'
200+
needs: windows_test
201+
steps:
202+
- name: Checkout repository
203+
uses: actions/checkout@v3
204+
- name: Get mmdeploy version
205+
run: |
206+
conda activate mmdeploy-3.8
207+
$env:MMDEPLOY_VERSION=(python -c "import sys; sys.path.append('mmdeploy');from version import __version__;print(__version__)")
208+
echo $env:MMDEPLOY_VERSION
209+
echo "MMDEPLOY_VERSION=$env:MMDEPLOY_VERSION" >> $env:GITHUB_ENV
210+
- name: Upload mmdeploy
211+
run: |
212+
cd "D:/DEPS/ciartifact/$env:MMDEPLOY_VERSION/mmdeploy"
213+
conda activate mmdeploy-3.8
214+
# twine upload * --repository testpypi -u __token__ -p ${{ secrets.test_pypi_password }}
215+
twine upload * -u __token__ -p ${{ secrets.pypi_password }}
216+
- name: Upload mmdeploy_python
217+
run: |
218+
cd "D:/DEPS/ciartifact/$env:MMDEPLOY_VERSION/mmdeploy_python"
219+
conda activate mmdeploy-3.8
220+
# twine upload * --repository testpypi -u __token__ -p ${{ secrets.test_pypi_password }}
221+
twine upload * -u __token__ -p ${{ secrets.pypi_password }}
222+
- name: Zip mmdeploy sdk
223+
run: |
224+
cd "D:/DEPS/ciartifact/$env:MMDEPLOY_VERSION/sdk"
225+
$folders = $(ls).Name
226+
foreach ($folder in $folders) {
227+
Compress-Archive -Path $folder -DestinationPath "$folder.zip"
228+
}
229+
- name: Upload mmdeploy sdk
230+
uses: softprops/action-gh-release@v1
231+
with:
232+
files: |
233+
D:/DEPS/ciartifact/$env:MMDEPLOY_VERSION/sdk/*.zip

‎.pre-commit-config.yaml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,9 @@ repos:
33
rev: 4.0.1
44
hooks:
55
- id: flake8
6-
args: ["--exclude=*/client/inference_pb2.py,*/client/inference_pb2_grpc.py"]
6+
args: ["--exclude=*/client/inference_pb2.py, \
7+
*/client/inference_pb2_grpc.py, \
8+
tools/package_tools/packaging/setup.py"]
79
- repo: https://github.com/PyCQA/isort
810
rev: 5.11.5
911
hooks:

‎MANIFEST.in‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ include mmdeploy/backend/ncnn/*.dll
44
include mmdeploy/backend/ncnn/*.pyd
55
include mmdeploy/backend/ncnn/mmdeploy_onnx2ncnn*
66
include mmdeploy/lib/*.so
7+
include mmdeploy/lib/*.so*
78
include mmdeploy/lib/*.dll
89
include mmdeploy/lib/*.pyd
910
include mmdeploy/backend/torchscript/*.so

‎cmake/loader.cpp.in‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,27 @@
2323
namespace mmdeploy {
2424
namespace {
2525

26+
#ifdef _WIN32
27+
inline static const std::wstring GetDllPath() {
28+
HMODULE hm = NULL;
29+
GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
30+
(LPWSTR)&GetDllPath, &hm);
31+
std::wstring ret;
32+
ret.resize(MAX_PATH);
33+
GetModuleFileNameW(hm, &ret[0], ret.size());
34+
ret = ret.substr(0, ret.find_last_of(L"/\\"));
35+
return ret;
36+
}
37+
#endif
38+
2639
void* mmdeploy_load_library(const char* name) {
2740
fprintf(stderr, "loading %s ...\n", name);
2841

2942
#ifdef _WIN32
30-
auto handle = LoadLibraryExA(name, NULL, LOAD_WITH_ALTERED_SEARCH_PATH);
43+
auto handle = LoadLibraryExA(name, NULL, LOAD_LIBRARY_SEARCH_USER_DIRS);
44+
if (handle == NULL) {
45+
handle = LoadLibraryExA(name, NULL, LOAD_WITH_ALTERED_SEARCH_PATH);
46+
}
3147
#else
3248
auto handle = dlopen(name, RTLD_NOW | RTLD_GLOBAL);
3349
#endif
@@ -44,6 +60,9 @@ void* mmdeploy_load_library(const char* name) {
4460
class Loader {
4561
public:
4662
Loader() {
63+
#ifdef _WIN32
64+
AddDllDirectory(GetDllPath().c_str());
65+
#endif
4766
const char* modules[] = {
4867
@_MMDEPLOY_DYNAMIC_MODULES@
4968
};

‎csrc/mmdeploy/apis/python/CMakeLists.txt‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,15 @@ endforeach ()
2626
pybind11_add_module(${PROJECT_NAME} ${MMDEPLOY_PYTHON_SRCS})
2727
# disable MMDEPLOY_CXX_USE_OPENCV in apis/cxx/mmdeploy/common.hpp
2828
target_compile_definitions(${PROJECT_NAME} PRIVATE -DMMDEPLOY_CXX_USE_OPENCV=0)
29+
if (APPLE)
30+
set_target_properties(${PROJECT_NAME} PROPERTIES
31+
BUILD_RPATH "@loader_path"
32+
INSTALL_RPATH "@loader_path")
33+
else ()
34+
set_target_properties(${PROJECT_NAME} PROPERTIES
35+
BUILD_RPATH "\$ORIGIN"
36+
INSTALL_RPATH "\$ORIGIN")
37+
endif ()
2938

3039
# https://github.com/pybind/pybind11/issues/1604
3140
if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")

‎csrc/mmdeploy/backend_ops/onnxruntime/CMakeLists.txt‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,15 @@ target_link_libraries(${PROJECT_NAME}_obj PUBLIC onnxruntime)
2222

2323
mmdeploy_add_library(${PROJECT_NAME} SHARED EXCLUDE "")
2424
target_link_libraries(${PROJECT_NAME} PUBLIC ${PROJECT_NAME}_obj)
25+
if (APPLE)
26+
set_target_properties(${PROJECT_NAME} PROPERTIES
27+
BUILD_RPATH "@loader_path"
28+
INSTALL_RPATH "@loader_path")
29+
else ()
30+
set_target_properties(${PROJECT_NAME} PROPERTIES
31+
BUILD_RPATH "\$ORIGIN"
32+
INSTALL_RPATH "\$ORIGIN")
33+
endif ()
2534
add_library(mmdeploy::onnxruntime::ops ALIAS ${PROJECT_NAME})
2635

2736
set(_ORT_OPS_DIR ${CMAKE_SOURCE_DIR}/mmdeploy/lib)

0 commit comments

Comments
 (0)