Skip to content

Commit 703c496

Browse files
velconiavelconia
committed
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into fix_reader_op_in_pserver
2 parents 5ebf56b + 982dabe commit 703c496

File tree

733 files changed

+2262
-840
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

733 files changed

+2262
-840
lines changed

‎CMakeLists.txt

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,10 @@ option(EIGEN_USE_THREADS "Compile with multi-threaded Eigen" OFF)
6161
option(WITH_ARM_FP16 "Use half precision support on armv8.2-a cpu" OFF)
6262
option(WITH_FAST_BUNDLE_TEST "Bundle tests that can be run in a single process together to reduce launch overhead" OFF)
6363
option(WITH_CONTRIB "Compile the third-party contributation" OFF)
64+
option(REPLACE_ENFORCE_GLOG "Replace PADDLE_ENFORCE with glog/CHECK for better debug." OFF)
6465
option(WITH_ANAKIN "Compile with Anakin library" OFF)
6566
option(WITH_GRPC "Use grpc as the default rpc framework" ${WITH_DISTRIBUTE})
67+
option(WITH_BRPC_RDMA "Use brpc rdma as the rpc protocal" OFF)
6668

6769
# CMAKE_BUILD_TYPE
6870
if(NOT CMAKE_BUILD_TYPE)
@@ -131,6 +133,10 @@ if (NOT DEFINED WITH_MKLDNN)
131133
set(WITH_MKLDNN OFF)
132134
endif()
133135
endif()
136+
137+
if (REPLACE_ENFORCE_GLOG)
138+
add_definitions("-DREPLACE_ENFORCE_GLOG")
139+
endif()
134140
########################################################################################
135141

136142
include(external/mklml) # download mklml package
@@ -153,12 +159,24 @@ include(external/cares)
153159
if(WITH_DISTRIBUTE)
154160
if(WITH_GRPC)
155161
include(external/grpc)
162+
message(STATUS "Use grpc framework.")
156163
else()
164+
message(STATUS "Use brpc framework.")
157165
include(external/leveldb)
158166
include(external/brpc)
159167
endif()
160168
endif()
161169

170+
if(WITH_BRPC_RDMA)
171+
message(STATUS "Use brpc with rdma.")
172+
if(WITH_GRPC)
173+
message(FATAL_ERROR "Can't use grpc with brpc rdma.")
174+
endif()
175+
if(NOT WITH_DISTRIBUTE)
176+
message(FATAL_ERROR "Can't use brpc rdma in no distribute env.")
177+
endif()
178+
endif()
179+
162180
include(external/snappy) # download snappy
163181
include(external/snappystream)
164182
include(external/threadpool)
@@ -178,7 +196,7 @@ include(inference_lib) # add paddle fluid inference libraries
178196

179197

180198
include_directories("${PADDLE_SOURCE_DIR}")
181-
include_directories("${PADDLE_SOURCE_DIR}/paddle/cuda/include")
199+
include_directories("${PADDLE_SOURCE_DIR}/paddle/legacy/cuda/include")
182200
include_directories("${CMAKE_CURRENT_BINARY_DIR}/proto")
183201
include_directories("${CMAKE_CURRENT_BINARY_DIR}/go/pserver/client/c")
184202

@@ -222,7 +240,7 @@ add_subdirectory(proto)
222240
if(NOT MOBILE_INFERENCE AND NOT WITH_FLUID_ONLY)
223241
# "add_subdirectory(go)" should be placed after the following loine,
224242
# because it depends on paddle/optimizer.
225-
add_subdirectory(paddle/optimizer)
243+
add_subdirectory(paddle/legacy/optimizer)
226244
endif()
227245

228246
# "add_subdirectory(paddle)" and "add_subdirectory(python)" should be

‎CONTRIBUTING.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -159,4 +159,4 @@ This will enable VLOG messages generated by `buddy_allocator.{h,cc}` and in the
159159
- verbose level 1: [framework](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/framework)
160160
- verbose level 3: [operators](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/operators)
161161
- verbose level 5: [memory](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/memory), [platform](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/platform)
162-
- verbose level 7: [math](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/math)
162+
- verbose level 7: [math](https://github.com/PaddlePaddle/Paddle/tree/develop/paddle/legacy/math)

‎README.md

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
[![Build Status](https://travis-ci.org/PaddlePaddle/Paddle.svg?branch=develop)](https://travis-ci.org/PaddlePaddle/Paddle)
55
[![Documentation Status](https://img.shields.io/badge/docs-latest-brightgreen.svg?style=flat)](http://www.paddlepaddle.org/docs/develop/documentation/en/getstarted/index_en.html)
66
[![Documentation Status](https://img.shields.io/badge/中文文档-最新-brightgreen.svg)](http://www.paddlepaddle.org/docs/develop/documentation/zh/getstarted/index_cn.html)
7-
[![Coverage Status](https://coveralls.io/repos/github/PaddlePaddle/Paddle/badge.svg?branch=develop)](https://coveralls.io/github/PaddlePaddle/Paddle?branch=develop)
87
[![Release](https://img.shields.io/github/release/PaddlePaddle/Paddle.svg)](https://github.com/PaddlePaddle/Paddle/releases)
98
[![License](https://img.shields.io/badge/license-Apache%202-blue.svg)](LICENSE)
109

‎cmake/configure.cmake

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,3 +174,7 @@ endif(WITH_GOLANG)
174174
if(WITH_GRPC)
175175
add_definitions(-DPADDLE_WITH_GRPC)
176176
endif(WITH_GRPC)
177+
178+
if(WITH_BRPC_RDMA)
179+
add_definitions(-DPADDLE_WITH_BRPC_RDMA)
180+
endif(WITH_BRPC_RDMA)

‎cmake/external/brpc.cmake

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,15 @@
1414

1515
INCLUDE(ExternalProject)
1616

17+
find_library(SSL_LIBRARY NAMES ssl)
18+
ADD_LIBRARY(ssl SHARED IMPORTED GLOBAL)
19+
SET_PROPERTY(TARGET ssl PROPERTY IMPORTED_LOCATION ${SSL_LIBRARY})
20+
21+
find_library(CRYPTO_LIBRARY NAMES crypto)
22+
ADD_LIBRARY(crypto SHARED IMPORTED GLOBAL)
23+
SET_PROPERTY(TARGET crypto PROPERTY IMPORTED_LOCATION ${CRYPTO_LIBRARY})
24+
25+
1726
SET(BRPC_SOURCES_DIR ${THIRD_PARTY_PATH}/brpc)
1827
SET(BRPC_INSTALL_DIR ${THIRD_PARTY_PATH}/install/brpc)
1928
SET(BRPC_INCLUDE_DIR "${BRPC_INSTALL_DIR}/include" CACHE PATH "brpc include directory." FORCE)
@@ -22,14 +31,14 @@ SET(BRPC_LIBRARIES "${BRPC_INSTALL_DIR}/lib/libbrpc.a" CACHE FILEPATH "brpc libr
2231
INCLUDE_DIRECTORIES(${BRPC_INCLUDE_DIR})
2332

2433
# Reference https://stackoverflow.com/questions/45414507/pass-a-list-of-prefix-paths-to-externalproject-add-in-cmake-args
25-
set(prefix_path "${THIRD_PARTY_PATH}/install/gflags|${THIRD_PARTY_PATH}/install/leveldb|${THIRD_PARTY_PATH}/install/snappy|${THIRD_PARTY_PATH}/install/gtest|${THIRD_PARTY_PATH}/install/protobuf")
34+
set(prefix_path "${THIRD_PARTY_PATH}/install/gflags|${THIRD_PARTY_PATH}/install/leveldb|${THIRD_PARTY_PATH}/install/snappy|${THIRD_PARTY_PATH}/install/gtest|${THIRD_PARTY_PATH}/install/protobuf|${THIRD_PARTY_PATH}/install/zlib")
2635

2736
# If minimal .a is need, you can set WITH_DEBUG_SYMBOLS=OFF
2837
ExternalProject_Add(
2938
extern_brpc
3039
${EXTERNAL_PROJECT_LOG_ARGS}
31-
GIT_REPOSITORY "https://github.com/brpc/brpc"
32-
GIT_TAG "6d153dd7ff00f960ae6895c9c5fff0ce9f07aff2"
40+
GIT_REPOSITORY "https://github.com/gongweibao/brpc"
41+
GIT_TAG "7dc04defad1fd4173aae170c3fcbde131b65155a"
3342
PREFIX ${BRPC_SOURCES_DIR}
3443
UPDATE_COMMAND ""
3544
CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
@@ -42,14 +51,16 @@ ExternalProject_Add(
4251
-DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
4352
-DCMAKE_PREFIX_PATH=${prefix_path}
4453
-DBRPC_WITH_GLOG=ON
54+
-DIOBUF_WITH_HUGE_BLOCK=ON
55+
-DBRPC_WITH_RDMA=${WITH_BRPC_RDMA}
4556
${EXTERNAL_OPTIONAL_ARGS}
4657
LIST_SEPARATOR |
4758
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${BRPC_INSTALL_DIR}
4859
-DCMAKE_INSTALL_LIBDIR:PATH=${BRPC_INSTALL_DIR}/lib
4960
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
5061
-DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
5162
)
52-
ADD_DEPENDENCIES(extern_brpc protobuf leveldb gflags glog gtest snappy)
63+
ADD_DEPENDENCIES(extern_brpc protobuf ssl crypto leveldb gflags glog gtest snappy)
5364
ADD_LIBRARY(brpc STATIC IMPORTED GLOBAL)
5465
SET_PROPERTY(TARGET brpc PROPERTY IMPORTED_LOCATION ${BRPC_LIBRARIES})
5566
ADD_DEPENDENCIES(brpc extern_brpc)

‎cmake/generic.cmake

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,20 @@ if(NOT APPLE AND NOT ANDROID)
9696
set(CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE} -pthread -ldl -lrt")
9797
endif(NOT APPLE AND NOT ANDROID)
9898

99+
set_property(GLOBAL PROPERTY FLUID_MODULES "")
100+
# find all fluid modules is used for paddle fluid static library
101+
# for building inference libs
102+
function(find_fluid_modules TARGET_NAME)
103+
get_filename_component(__target_path ${TARGET_NAME} ABSOLUTE)
104+
string(REGEX REPLACE "^${PADDLE_SOURCE_DIR}/" "" __target_path ${__target_path})
105+
string(FIND "${__target_path}" "fluid" pos)
106+
if(pos GREATER 1)
107+
get_property(fluid_modules GLOBAL PROPERTY FLUID_MODULES)
108+
set(fluid_modules ${fluid_modules} ${TARGET_NAME})
109+
set_property(GLOBAL PROPERTY FLUID_MODULES "${fluid_modules}")
110+
endif()
111+
endfunction(find_fluid_modules)
112+
99113
function(merge_static_libs TARGET_NAME)
100114
set(libs ${ARGN})
101115
list(REMOVE_DUPLICATES libs)
@@ -250,6 +264,7 @@ function(cc_test TARGET_NAME)
250264
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
251265
if (${cc_test_SERIAL})
252266
set_property(TEST ${TARGET_NAME} PROPERTY SERIAL 1)
267+
set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT FLAGS_init_allocated_mem=true)
253268
endif()
254269
endif()
255270
endfunction(cc_test)
@@ -314,6 +329,7 @@ function(nv_test TARGET_NAME)
314329
add_test(${TARGET_NAME} ${TARGET_NAME})
315330
if (nv_test_SERIAL)
316331
set_property(TEST ${TARGET_NAME} PROPERTY SERIAL 1)
332+
set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT FLAGS_init_allocated_mem=true)
317333
endif()
318334
endif()
319335
endfunction(nv_test)
@@ -561,7 +577,7 @@ function(py_test TARGET_NAME)
561577
set(multiValueArgs SRCS DEPS ARGS ENVS)
562578
cmake_parse_arguments(py_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
563579
add_test(NAME ${TARGET_NAME}
564-
COMMAND env PYTHONPATH=${PADDLE_BINARY_DIR}/python ${py_test_ENVS}
580+
COMMAND env FLAGS_init_allocated_mem=true PYTHONPATH=${PADDLE_BINARY_DIR}/python ${py_test_ENVS}
565581
${PYTHON_EXECUTABLE} -u ${py_test_SRCS} ${py_test_ARGS}
566582
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
567583
endif()

‎cmake/inference_lib.cmake

Lines changed: 3 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -12,19 +12,6 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
set_property(GLOBAL PROPERTY FLUID_MODULES "")
16-
# find all fluid modules is used for paddle fluid static library
17-
function(find_fluid_modules TARGET_NAME)
18-
get_filename_component(__target_path ${TARGET_NAME} ABSOLUTE)
19-
string(REGEX REPLACE "^${PADDLE_SOURCE_DIR}/" "" __target_path ${__target_path})
20-
string(FIND "${__target_path}" "fluid" pos)
21-
if(pos GREATER 1)
22-
get_property(fluid_modules GLOBAL PROPERTY FLUID_MODULES)
23-
set(fluid_modules ${fluid_modules} ${TARGET_NAME})
24-
set_property(GLOBAL PROPERTY FLUID_MODULES "${fluid_modules}")
25-
endif()
26-
endfunction(find_fluid_modules)
27-
2815
# make package for paddle fluid shared and static library
2916
function(copy TARGET)
3017
set(options "")
@@ -154,7 +141,7 @@ set(inference_deps paddle_fluid_shared paddle_fluid)
154141
if(WITH_CONTRIB)
155142
message(STATUS "installing contrib")
156143
set(contrib_dst_dir "${FLUID_INSTALL_DIR}/contrib/inference")
157-
if (WITH_ANAKIN)
144+
if (WITH_ANAKIN AND WITH_GPU)
158145
copy(contrib_anakin_inference_lib DEPS paddle_inference_api inference_anakin_api
159146
SRCS
160147
${PADDLE_BINARY_DIR}/paddle/contrib/inference/libinference_anakin_api* # compiled anakin api
@@ -163,9 +150,9 @@ if(WITH_CONTRIB)
163150
list(APPEND inference_deps contrib_anakin_inference_lib)
164151
endif()
165152

166-
copy(contrib_inference_lib DEPS paddle_inference_api
153+
copy(contrib_inference_lib DEPS paddle_inference_api paddle_inference_api_shared
167154
SRCS ${PADDLE_SOURCE_DIR}/paddle/contrib/inference/paddle_inference_api.h
168-
${PADDLE_BINARY_DIR}/paddle/contrib/inference/libpaddle_inference_api.*
155+
${PADDLE_BINARY_DIR}/paddle/contrib/inference/libpaddle_inference_api*
169156
DSTS ${contrib_dst_dir} ${contrib_dst_dir})
170157
list(APPEND inference_deps contrib_inference_lib)
171158
endif()

‎doc/fluid/api/layers.rst

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1468,6 +1468,14 @@ argmax
14681468
.. autofunction:: paddle.fluid.layers.argmax
14691469
:noindex:
14701470

1471+
.. _api_fluid_layers_argsort:
1472+
1473+
argsort
1474+
-------
1475+
1476+
.. autofunction:: paddle.fluid.layers.argsort
1477+
:noindex:
1478+
14711479
.. _api_fluid_layers_ones:
14721480

14731481
ones

‎doc/fluid/design/multi_devices/kernel_selection.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,10 +74,10 @@ void OperatorWithKernel::Run(
7474
auto kernel_type_for_var = this->GetKernelTypeForVar(...);
7575
if (kernel_type_for_var.place_ != expected_kernel_key.place_) {
7676
auto* trans_var = new_scope.Var(var_name);
77-
auto* out = DataTransform(expected_kernel_key,
77+
auto* out = TransformData(expected_kernel_key,
7878
kernel_type_for_var,
7979
*tensor_in);
80-
CopyVariableWithTensor(...);
80+
SetTensorToVariable(...);
8181
}
8282
}
8383

‎doc/v2/design/interface/00.why_plain_c.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ paddle_error paddle_matrix_get_shape(paddle_matrix matrix,
6565
而在CPP里面实现这个C的接口，文件 `paddle_matrix.cpp`
6666
6767
```cpp
68-
#include "paddle/math/matrix.h"
68+
#include "paddle/legacy/math/matrix.h"
6969
extern "C"
7070
paddle_error paddle_matrix_shape(paddle_matrix matrix,
7171
uint64_t *width,

0 commit comments

Comments
 (0)