Skip to content

Commit 3ba8c48

Browse files
zhwesky2010zhwesky2010
authored
[CHERRY-PICK2.1]Remove paddle_custom_op dynamic libraries, and link to FLUID_CORE on windows (#32583) (#32769)
* Remove paddle_custom_op dynamic libraries, change link to FLUID_CORE on windows, and check copy_to * fix CI
1 parent 70e0e3d commit 3ba8c48

File tree

13 files changed

+405
-175
lines changed

13 files changed

+405
-175
lines changed

‎paddle/fluid/framework/CMakeLists.txt

Lines changed: 0 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -369,36 +369,3 @@ cc_library(paddle_framework DEPS ${FLUID_FRAMEWORK_MODULES})
369369
if(WITH_TESTING AND TEST selected_rows_test)
370370
set_tests_properties(selected_rows_test PROPERTIES TIMEOUT 120)
371371
endif()
372-
373-
##### 2.0 New custom op extension mechanism related #####
374-
375-
# if not deps `layer`, will cause: undefined symbol: _ZN6paddle10imperative7VarBase9name_set_
376-
if (WIN32)
377-
set(PADDLE_CUSTOM_OP_MODULES custom_tensor op_meta_info custom_operator layer)
378-
379-
set(PADDLE_CUSTOM_OP_SRCS
380-
${CMAKE_CURRENT_SOURCE_DIR}/custom_operator.cc
381-
${CMAKE_CURRENT_SOURCE_DIR}/../extension/src/ext_tensor.cc
382-
${CMAKE_CURRENT_SOURCE_DIR}/../extension/src/ext_op_meta_info.cc
383-
${CMAKE_SOURCE_DIR}/paddle/fluid/imperative/layer.cc)
384-
set(PADDLE_CUSTOM_OP_SRCS ${PADDLE_CUSTOM_OP_SRCS} PARENT_SCOPE)
385-
386-
cc_library(paddle_custom_op_shared
387-
SHARED SRCS ${PADDLE_CUSTOM_OP_SRCS} DEPS ${PADDLE_CUSTOM_OP_MODULES})
388-
389-
get_property(os_dependency_modules GLOBAL PROPERTY OS_DEPENDENCY_MODULES)
390-
set_target_properties(paddle_custom_op_shared PROPERTIES OUTPUT_NAME paddle_custom_op)
391-
target_link_libraries(paddle_custom_op_shared ${os_dependency_modules})
392-
393-
if("${CMAKE_GENERATOR}" STREQUAL "Ninja")
394-
set(paddle_custom_op_lib_path ${CMAKE_CURRENT_BINARY_DIR})
395-
else()
396-
set(paddle_custom_op_lib_path ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE})
397-
endif()
398-
set(PADDLE_CUSTOM_OP_IMPORT_LIB
399-
${paddle_custom_op_lib_path}/paddle_custom_op.lib
400-
CACHE INTERNAL "Paddle custom op import lib")
401-
set(PADDLE_CUSTOM_OP_SHARED_LIB
402-
${paddle_custom_op_lib_path}/paddle_custom_op.dll
403-
CACHE INTERNAL "Paddle custom op dll")
404-
endif()

‎paddle/scripts/paddle_build.bat

Lines changed: 52 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -54,14 +54,14 @@ wmic process where name="python.exe" call terminate 2>NUL
5454
rem ------initialize common variable------
5555
if not defined GENERATOR set GENERATOR="Visual Studio 15 2017 Win64"
5656
if not defined BRANCH set BRANCH=develop
57-
if not defined WITH_TENSORRT set WITH_TENSORRT=ON
57+
if not defined WITH_TENSORRT set WITH_TENSORRT=ON
5858
if not defined TENSORRT_ROOT set TENSORRT_ROOT=D:/TensorRT
5959
if not defined CUDA_ARCH_NAME set CUDA_ARCH_NAME=Auto
6060
if not defined WITH_GPU set WITH_GPU=ON
6161
if not defined WITH_MKL set WITH_MKL=ON
6262
if not defined WITH_AVX set WITH_AVX=ON
6363
if not defined WITH_TESTING set WITH_TESTING=ON
64-
if not defined MSVC_STATIC_CRT set MSVC_STATIC_CRT=OFF
64+
if not defined MSVC_STATIC_CRT set MSVC_STATIC_CRT=ON
6565
if not defined WITH_PYTHON set WITH_PYTHON=ON
6666
if not defined ON_INFER set ON_INFER=ON
6767
if not defined WITH_INFERENCE_API_TEST set WITH_INFERENCE_API_TEST=ON
@@ -75,6 +75,7 @@ if not defined LOG_LEVEL set LOG_LEVEL=normal
7575
if not defined PRECISION_TEST set PRECISION_TEST=OFF
7676
if not defined NIGHTLY_MODE set PRECISION_TEST=OFF
7777
if not defined retry_times set retry_times=2
78+
if not defined PYTHON_ROOT set PYTHON_ROOT=C:\Python37
7879

7980
rem -------set cache build directory-----------
8081
rmdir build\python /s/q
@@ -83,9 +84,6 @@ rmdir build\paddle_inference_install_dir /s/q
8384
rmdir build\paddle_inference_c_install_dir /s/q
8485
del build\CMakeCache.txt
8586

86-
: set CI_SKIP_CPP_TEST if only *.py changed
87-
git diff --name-only %BRANCH% | findstr /V "\.py" || set CI_SKIP_CPP_TEST=ON
88-
8987
if "%WITH_CACHE%"=="OFF" (
9088
rmdir build /s/q
9189
goto :mkbuild
@@ -135,58 +133,6 @@ dir .
135133
dir %cache_dir%
136134
dir paddle\fluid\pybind\Release
137135

138-
rem ------initialize the python environment------
139-
if not defined PYTHON_ROOT set PYTHON_ROOT=C:\Python37
140-
set PYTHON_EXECUTABLE=%PYTHON_ROOT%\python.exe
141-
set PATH=%PYTHON_ROOT%;%PYTHON_ROOT%\Scripts;%PATH%
142-
143-
rem ToDo: virtual environment can't be deleted safely, some process not exit when task is canceled
144-
rem Now use system python environment temporarily
145-
rem %PYTHON_EXECUTABLE% -m pip install virtualenv
146-
rem %PYTHON_EXECUTABLE% -m virtualenv paddle_winci
147-
rem call paddle_winci\Scripts\activate.bat
148-
149-
rem ------pre install python requirement----------
150-
where python
151-
where pip
152-
pip install wheel --user
153-
pip install -r %work_dir%\python\requirements.txt --user
154-
155-
if %ERRORLEVEL% NEQ 0 (
156-
echo pip install requirements.txt failed!
157-
exit /b 7
158-
)
159-
160-
rem ------pre install clcache and init config----------
161-
rem pip install clcache --user
162-
pip uninstall -y clcache
163-
:: set USE_CLCACHE to enable clcache
164-
rem set USE_CLCACHE=1
165-
:: In some scenarios, CLCACHE_HARDLINK can save one file copy.
166-
rem set CLCACHE_HARDLINK=1
167-
:: If it takes more than 1000s to obtain the right to use the cache, an error will be reported
168-
rem set CLCACHE_OBJECT_CACHE_TIMEOUT_MS=1000000
169-
:: set maximum cache size to 20G
170-
rem clcache.exe -M 21474836480
171-
172-
:: install ninja if GENERATOR is Ninja
173-
if %GENERATOR% == "Ninja" (
174-
pip install ninja
175-
if %errorlevel% NEQ 0 (
176-
echo pip install ninja failed!
177-
exit /b 7
178-
)
179-
)
180-
181-
rem ------show summary of current environment----------
182-
cmake --version
183-
if "%WITH_GPU%"=="ON" (
184-
nvcc --version
185-
nvidia-smi
186-
)
187-
::python %work_dir%\tools\summary_env.py
188-
::%cache_dir%\tools\busybox64.exe bash %work_dir%\tools\get_cpu_info.sh
189-
190136
goto :CASE_%1
191137

192138
echo "Usage: paddle_build.bat [OPTION]"
@@ -266,8 +212,10 @@ rem "Other configurations are added here"
266212
rem :CASE_wincheck_others
267213
rem call ...
268214

215+
269216
rem ---------------------------------------------------------------------------------------------
270217
:cmake
218+
@ECHO OFF
271219
echo ========================================
272220
echo Step 1. Cmake ...
273221
echo ========================================
@@ -281,12 +229,52 @@ set PATH=C:\Program Files (x86)\Windows Kits\10\bin\10.0.17763.0\x64;%PATH%
281229
for /F %%# in ('wmic os get localdatetime^|findstr 20') do set start=%%#
282230
set start=%start:~4,10%
283231

284-
@ECHO ON
285-
if not defined CUDA_TOOLKIT_ROOT_DIR set CUDA_TOOLKIT_ROOT_DIR=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.0
232+
if not defined CUDA_TOOLKIT_ROOT_DIR set CUDA_TOOLKIT_ROOT_DIR=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2
286233
set PATH=%TENSORRT_ROOT:/=\%\lib;%CUDA_TOOLKIT_ROOT_DIR%\bin;%CUDA_TOOLKIT_ROOT_DIR%\libnvvp;%PATH%
287234

288-
rem ------set third_party cache dir------
235+
rem install ninja if GENERATOR is Ninja
236+
if %GENERATOR% == "Ninja" (
237+
pip install ninja
238+
if %errorlevel% NEQ 0 (
239+
echo pip install ninja failed!
240+
exit /b 7
241+
)
242+
)
289243

244+
rem ------show summary of current GPU environment----------
245+
cmake --version
246+
if "%WITH_GPU%"=="ON" (
247+
nvcc --version
248+
nvidia-smi
249+
)
250+
251+
rem ------initialize the python environment------
252+
set PYTHON_EXECUTABLE=%PYTHON_ROOT%\python.exe
253+
set PATH=%PYTHON_ROOT%;%PYTHON_ROOT%\Scripts;%PATH%
254+
if %WITH_PYTHON% == "OFF" (
255+
where python
256+
where pip
257+
pip install wheel --user
258+
pip install -r %work_dir%\python\requirements.txt --user
259+
if %ERRORLEVEL% NEQ 0 (
260+
echo pip install requirements.txt failed!
261+
exit /b 7
262+
)
263+
)
264+
265+
rem ------pre install clcache and init config----------
266+
rem pip install clcache --user
267+
pip uninstall -y clcache
268+
:: set USE_CLCACHE to enable clcache
269+
rem set USE_CLCACHE=1
270+
:: In some scenarios, CLCACHE_HARDLINK can save one file copy.
271+
rem set CLCACHE_HARDLINK=1
272+
:: If it takes more than 1000s to obtain the right to use the cache, an error will be reported
273+
rem set CLCACHE_OBJECT_CACHE_TIMEOUT_MS=1000000
274+
:: set maximum cache size to 20G
275+
rem clcache.exe -M 21474836480
276+
277+
rem ------set third_party cache dir------
290278
: clear third party cache every once in a while
291279
for /F %%# in ('wmic os get localdatetime^|findstr 20') do set datetime=%%#
292280
set day_now=%datetime:~6,2%
@@ -500,6 +488,10 @@ echo ========================================
500488
echo Step 4. Running unit tests ...
501489
echo ========================================
502490

491+
492+
: set CI_SKIP_CPP_TEST if only *.py changed
493+
git diff --name-only %BRANCH% | findstr /V "\.py" || set CI_SKIP_CPP_TEST=ON
494+
503495
pip install -r %work_dir%\python\unittest_py\requirements.txt --user
504496
if %ERRORLEVEL% NEQ 0 (
505497
echo pip install unittest requirements.txt failed!

‎python/CMakeLists.txt

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,20 @@ set(FLUID_DST_DIR ${PADDLE_BINARY_DIR}/python/paddle/fluid/)
4343
IF(WIN32)
4444
# Python would use the .pyd by default under Windows series platform
4545
set(FLUID_CORE ${FLUID_DST_DIR}/${FLUID_CORE_NAME}.pyd)
46-
set(FLUID_NOAVX_CORE ${FLUID_DST_DIR}/core_noavx.pyd)
46+
set(FLUID_CORE_LIB ${FLUID_DST_DIR}/${FLUID_CORE_NAME}.lib)
47+
48+
add_custom_command(OUTPUT ${FLUID_CORE}
49+
COMMAND cmake -E copy $<TARGET_FILE:paddle_pybind> ${FLUID_CORE}
50+
COMMAND cmake -E copy $<TARGET_LINKER_FILE:paddle_pybind> ${FLUID_CORE_LIB}
51+
DEPENDS paddle_pybind)
52+
53+
set(FLUID_NOAVX_CORE ${FLUID_DST_DIR}/core_noavx.pyd)
4754
ELSE()
4855
set(FLUID_CORE ${FLUID_DST_DIR}/${FLUID_CORE_NAME}.so)
56+
add_custom_command(OUTPUT ${FLUID_CORE}
57+
COMMAND cmake -E copy $<TARGET_FILE:paddle_pybind> ${FLUID_CORE}
58+
DEPENDS paddle_pybind)
59+
4960
set(FLUID_NOAVX_CORE ${FLUID_DST_DIR}/core_noavx.so)
5061
ENDIF()
5162

@@ -68,9 +79,6 @@ if(HAS_NOAVX_CORE AND EXISTS "${NOAVX_CORE_FILE}")
6879
list(APPEND FLUID_CORE_DEPS ${FLUID_NOAVX_CORE})
6980
endif()
7081

71-
add_custom_command(OUTPUT ${FLUID_CORE}
72-
COMMAND cmake -E copy $<TARGET_FILE:paddle_pybind> ${FLUID_CORE}
73-
DEPENDS paddle_pybind)
7482
add_custom_target(copy_paddle_pybind ALL DEPENDS ${FLUID_CORE_DEPS})
7583

7684
IF(WIN32)
@@ -84,6 +92,7 @@ ELSE(WIN32)
8492
COMMAND touch stub.cc
8593
COMMAND cp -r ${PADDLE_SOURCE_DIR}/python/paddle ${PADDLE_BINARY_DIR}/python
8694
COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel
95+
COMMENT "Packing whl packages------>>>"
8796
DEPENDS copy_paddle_pybind ${FLUID_CORE} framework_py_proto profiler_py_proto ${PY_FILES})
8897
ENDIF()
8998

‎python/paddle/check_import_scipy.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,6 @@ def check_import_scipy(OsName):
2424
if 'DLL load failed' in print_info:
2525
raise ImportError(
2626
print_info +
27-
"\nplease download visual C++ Redistributable for vs 2015, https://www.microsoft.com/en-us/download/details.aspx?id=48145"
27+
"\nplease download Visual C++ Redistributable from https://support.microsoft.com/en-us/topic/the-latest-supported-visual-c-downloads-2647da03-1eea-4433-9aff-95f26a218cc0"
2828
)
2929
return

‎python/paddle/fluid/core.py

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,10 @@
3737
try:
3838
if os.name == 'nt':
3939
third_lib_path = current_path + os.sep + '..' + os.sep + 'libs'
40-
os.environ['path'] = third_lib_path + ';' + os.environ['path']
40+
# Will load shared library from 'path' on windows
41+
os.environ[
42+
'path'] = current_path + ';' + third_lib_path + ';' + os.environ[
43+
'path']
4144
sys.path.insert(0, third_lib_path)
4245
# Note: from python3.8, PATH will not take effect
4346
# https://github.com/python/cpython/pull/12302
@@ -298,7 +301,7 @@ def to_list(s):
298301
"WARNING: AVX is supported on local machine, but you have installed "
299302
"paddlepaddle without avx core. Hence, no_avx core which has worse "
300303
"preformance will be imported.\nYou could reinstall paddlepaddle by "
301-
"'python -m pip install -U paddlepaddle-gpu[==version]' or rebuild "
304+
"'python -m pip install --force-reinstall paddlepaddle-gpu[==version]' or rebuild "
302305
"paddlepaddle WITH_AVX=ON to get better performance.\n"
303306
"The original error is: %s\n" % cpt.get_exception_message(e))
304307
load_noavx = True
@@ -350,12 +353,19 @@ def to_list(s):
350353
sys.stderr.write(
351354
'Error: Can not import noavx core while this file exists: ' +
352355
current_path + os.sep + 'core_noavx.' + core_suffix + '\n')
356+
elif avx_supported():
357+
sys.stderr.write(
358+
"Error: AVX is support on your machine, but you have installed "
359+
"paddlepaddle without avx core, you should reinstall paddlepaddle by "
360+
"'python -m pip install --force-reinstall paddlepaddle-gpu[==version]\n"
361+
)
353362
else:
354363
sys.stderr.write(
355364
"Error: AVX is not support on your machine, but you have installed "
356-
"paddlepaddle with avx core, you should reinstall paddlepaddle by "
357-
"'python -m pip install -U paddlepaddle-gpu[==version] -f "
358-
"https://paddlepaddle.org.cn/whl/stable_noavx.html'\n")
365+
"paddlepaddle without no_avx core, you should reinstall paddlepaddle by "
366+
"'python -m pip install --force-reinstall paddlepaddle-gpu[==version] -f "
367+
"https://paddlepaddle.org.cn/whl/mkl/stable/noavx.html or "
368+
"https://paddlepaddle.org.cn/whl/openblas/stable/noavx.html\n")
359369
raise e
360370

361371

‎python/paddle/fluid/tests/custom_op/CMakeLists.txt

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
1-
# New custom OP can support Windows/Linux now
2-
if(WITH_GPU OR APPLE)
3-
# GPU custom op tests: compile both .cc and .cu file
1+
# New custom OP can support Windows/Linux/Mac now
2+
if(WITH_GPU OR APPLE)
43
py_test(test_custom_relu_op_setup SRCS test_custom_relu_op_setup.py)
54
py_test(test_custom_relu_op_jit SRCS test_custom_relu_op_jit.py)
65
py_test(test_custom_relu_model SRCS test_custom_relu_model.py)

‎python/paddle/fluid/tests/custom_op/custom_relu_op.cu

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,12 @@ std::vector<paddle::Tensor> relu_cuda_forward(const paddle::Tensor& x) {
4545
int grid = (numel + block - 1) / block;
4646
PD_DISPATCH_FLOATING_AND_HALF_TYPES(
4747
x.type(), "relu_cuda_forward_kernel", ([&] {
48+
auto cpu_input = x.copy_to<data_t>(paddle::PlaceType::kCPU);
49+
auto gpu_input = cpu_input.copy_to<data_t>(paddle::PlaceType::kGPU);
4850
relu_cuda_forward_kernel<data_t><<<grid, block, 0, x.stream()>>>(
49-
x.data<data_t>(), out.mutable_data<data_t>(x.place()), numel);
51+
gpu_input.data<data_t>(),
52+
out.mutable_data<data_t>(x.place()),
53+
numel);
5054
}));
5155

5256
return {out};

‎python/paddle/fluid/tests/custom_op/test_check_abi.py

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -64,14 +64,29 @@ def test_wrong_compiler_warning(self):
6464
# clear environ
6565
self.del_environ()
6666
compiler = 'python' # fake wrong compiler
67-
with warnings.catch_warnings(record=True) as error:
68-
flag = utils.check_abi_compatibility(compiler, verbose=True)
69-
# check return False
70-
self.assertFalse(flag)
71-
# check Compiler Compatibility WARNING
72-
self.assertTrue(len(error) == 1)
73-
self.assertTrue(
74-
"Compiler Compatibility WARNING" in str(error[0].message))
67+
if not utils.IS_WINDOWS:
68+
with warnings.catch_warnings(record=True) as error:
69+
flag = utils.check_abi_compatibility(compiler, verbose=True)
70+
# check return False
71+
self.assertFalse(flag)
72+
# check Compiler Compatibility WARNING
73+
self.assertTrue(len(error) == 1)
74+
self.assertTrue(
75+
"Compiler Compatibility WARNING" in str(error[0].message))
76+
77+
def test_exception_windows(self):
78+
# clear environ
79+
self.del_environ()
80+
compiler = 'fake compiler' # fake command
81+
if utils.IS_WINDOWS:
82+
with warnings.catch_warnings(record=True) as error:
83+
flag = utils.check_abi_compatibility(compiler, verbose=True)
84+
# check return False
85+
self.assertFalse(flag)
86+
# check ABI Compatibility WARNING
87+
self.assertTrue(len(error) == 1)
88+
self.assertTrue("Failed to check compiler version for" in
89+
str(error[0].message))
7590

7691
def test_exception_linux(self):
7792
# clear environ

‎python/paddle/fluid/tests/custom_op/test_custom_relu_op_jit.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -105,12 +105,12 @@ def test_exception(self):
105105
in str(e))
106106
if IS_WINDOWS:
107107
self.assertTrue(
108-
r"python\paddle\fluid\tests\custom_op\custom_relu_op.cc:47"
109-
in str(e))
108+
r"python\paddle\fluid\tests\custom_op\custom_relu_op.cc" in
109+
str(e))
110110
else:
111111
self.assertTrue(
112-
"python/paddle/fluid/tests/custom_op/custom_relu_op.cc:47"
113-
in str(e))
112+
"python/paddle/fluid/tests/custom_op/custom_relu_op.cc" in
113+
str(e))
114114
self.assertTrue(caught_exception)
115115

116116
caught_exception = False
@@ -126,7 +126,7 @@ def test_exception(self):
126126
"function \"relu_cuda_forward_kernel\" is not implemented for data type `int32_t`"
127127
in str(e))
128128
self.assertTrue(
129-
"python/paddle/fluid/tests/custom_op/custom_relu_op.cu:50" in
129+
"python/paddle/fluid/tests/custom_op/custom_relu_op.cu" in
130130
str(e))
131131
self.assertTrue(caught_exception)
132132

0 commit comments

Comments
 (0)