Skip to content

Commit 0a8a86e

Browse files
typhoonzerotyphoonzero
authored
Send recv op (#5520)
* WIP send recv op * WIP send recv * put grpc impl in details * put grpc impl in details * update wip * update proto * update proto * update proto * clean cmake * wip on op implementations * wip on op implementations * compile ok adding ut * wip unitest * add extern cares for linking * wip add ut * working version send recv * revert optimizer.py * update test cmake * add libtool to dockerfile * update cmake dependency * update cmake depends * update cmake grpc depends * fix cmake dependency * fix compile error * fix compile * follow comments * update * update copyfrom
1 parent dc82a30 commit 0a8a86e

21 files changed

+941
-142
lines changed

‎.clang-format

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,4 +25,3 @@ AllowAllParametersOfDeclarationOnNextLine: true
2525
BinPackParameters: false
2626
BinPackArguments: false
2727
...
28-

‎CMakeLists.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,8 @@ include(external/any) # download libn::any
133133
include(external/eigen) # download eigen3
134134
include(external/pybind11) # download pybind11
135135
include(external/nccl)
136+
include(external/cares)
137+
include(external/grpc)
136138

137139
include(cudnn) # set cudnn libraries, must before configure
138140
include(configure) # add paddle env configuration

‎Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ RUN apt-get update && \
2929
automake locales clang-format swig doxygen cmake \
3030
liblapack-dev liblapacke-dev libboost-dev \
3131
clang-3.8 llvm-3.8 libclang-3.8-dev \
32-
net-tools && \
32+
net-tools libtool && \
3333
apt-get clean -y
3434

3535
# Install Go and glide

‎cmake/external/cares.cmake

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
#
15+
16+
IF(MOBILE_INFERENCE)
17+
return()
18+
ENDIF()
19+
20+
include (ExternalProject)
21+
22+
# NOTE: c-ares is needed when linking with grpc.
23+
24+
SET(CARES_SOURCES_DIR ${THIRD_PARTY_PATH}/cares)
25+
SET(CARES_INSTALL_DIR ${THIRD_PARTY_PATH}/install/cares)
26+
SET(CARES_INCLUDE_DIR "${CARES_INSTALL_DIR}/include/" CACHE PATH "cares include directory." FORCE)
27+
28+
ExternalProject_Add(
29+
extern_cares
30+
GIT_REPOSITORY "https://github.com/c-ares/c-ares.git"
31+
GIT_TAG "cares-1_13_0"
32+
PREFIX ${CARES_SOURCES_DIR}
33+
UPDATE_COMMAND ""
34+
CONFIGURE_COMMAND ./buildconf && ./configure --disable-shared --prefix=${CARES_INSTALL_DIR}
35+
BUILD_IN_SOURCE 1
36+
BUILD_COMMAND make
37+
INSTALL_COMMAND make install
38+
)
39+
40+
ADD_LIBRARY(cares STATIC IMPORTED GLOBAL)
41+
SET_PROPERTY(TARGET cares PROPERTY IMPORTED_LOCATION
42+
"${CARES_INSTALL_DIR}/lib/libcares.a")
43+
44+
include_directories(${CARES_INCLUDE_DIR})
45+
ADD_DEPENDENCIES(cares extern_cares)

‎cmake/external/grpc.cmake

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
#
15+
16+
IF(MOBILE_INFERENCE)
17+
return()
18+
ENDIF()
19+
20+
include (ExternalProject)
21+
22+
SET(GRPC_SOURCES_DIR ${THIRD_PARTY_PATH}/grpc)
23+
SET(GRPC_INSTALL_DIR ${THIRD_PARTY_PATH}/install/grpc)
24+
SET(GRPC_INCLUDE_DIR "${GRPC_INSTALL_DIR}/include/" CACHE PATH "grpc include directory." FORCE)
25+
SET(GRPC_CPP_PLUGIN "${GRPC_INSTALL_DIR}/bin/grpc_cpp_plugin" CACHE FILEPATH "GRPC_CPP_PLUGIN" FORCE)
26+
27+
ExternalProject_Add(
28+
extern_grpc
29+
DEPENDS protobuf zlib
30+
GIT_REPOSITORY "https://github.com/grpc/grpc.git"
31+
GIT_TAG "v1.7.x"
32+
PREFIX ${GRPC_SOURCES_DIR}
33+
UPDATE_COMMAND ""
34+
CONFIGURE_COMMAND ""
35+
BUILD_IN_SOURCE 1
36+
BUILD_COMMAND make
37+
INSTALL_COMMAND make prefix=${GRPC_INSTALL_DIR} install
38+
)
39+
40+
# FIXME(typhoonzero): hack to get static lib path, try a better way like merge them.
41+
ADD_LIBRARY(grpc++_unsecure STATIC IMPORTED GLOBAL)
42+
SET_PROPERTY(TARGET grpc++_unsecure PROPERTY IMPORTED_LOCATION
43+
"${GRPC_INSTALL_DIR}/lib/libgrpc++_unsecure.a")
44+
45+
ADD_LIBRARY(grpc++ STATIC IMPORTED GLOBAL)
46+
SET_PROPERTY(TARGET grpc++ PROPERTY IMPORTED_LOCATION
47+
"${GRPC_INSTALL_DIR}/lib/libgrpc++.a")
48+
ADD_LIBRARY(gpr STATIC IMPORTED GLOBAL)
49+
SET_PROPERTY(TARGET gpr PROPERTY IMPORTED_LOCATION
50+
"${GRPC_INSTALL_DIR}/lib/libgpr.a")
51+
52+
ADD_LIBRARY(grpc_unsecure STATIC IMPORTED GLOBAL)
53+
SET_PROPERTY(TARGET grpc_unsecure PROPERTY IMPORTED_LOCATION
54+
"${GRPC_INSTALL_DIR}/lib/libgrpc_unsecure.a")
55+
56+
include_directories(${GRPC_INCLUDE_DIR})
57+
ADD_DEPENDENCIES(grpc++_unsecure extern_grpc)
58+

‎cmake/external/zlib.cmake

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ ExternalProject_Add(
5050
)
5151

5252
LIST(APPEND external_project_dependencies zlib)
53+
ADD_LIBRARY(zlib_target STATIC IMPORTED GLOBAL)
54+
SET_PROPERTY(TARGET zlib_target PROPERTY IMPORTED_LOCATION ${ZLIB_LIBRARIES})
5355

5456
IF(WITH_C_API)
5557
INSTALL(DIRECTORY ${ZLIB_INCLUDE_DIR} DESTINATION third_party/zlib)

‎cmake/generic.cmake

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -467,3 +467,50 @@ function(py_test TARGET_NAME)
467467
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
468468
endif()
469469
endfunction()
470+
471+
# grpc_library generate grpc code using grpc_cpp_plugin and protoc
472+
# then build the generated protobuf code and grpc code with your
473+
# implementation source codes together. Use SRCS argument for your
474+
# implementation source files and PROTO argument for your .proto
475+
# files.
476+
#
477+
# Usage: grpc_library(my_target SRCS my_client.cc PROTO my_target.proto DEPS my_dep)
478+
479+
function(grpc_library TARGET_NAME)
480+
set(oneValueArgs PROTO)
481+
set(multiValueArgs SRCS DEPS)
482+
set(options "")
483+
cmake_parse_arguments(grpc_library "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
484+
485+
message(STATUS "generating grpc ${grpc_library_PROTO}")
486+
487+
get_filename_component(ABS_PROTO ${grpc_library_PROTO} ABSOLUTE)
488+
get_filename_component(PROTO_WE ${grpc_library_PROTO} NAME_WE)
489+
get_filename_component(PROTO_PATH ${ABS_PROTO} PATH)
490+
491+
protobuf_generate_cpp(grpc_proto_srcs grpc_proto_hdrs "${ABS_PROTO}")
492+
set(grpc_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/${PROTO_WE}.grpc.pb.cc")
493+
set(grpc_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/${PROTO_WE}.grpc.pb.h")
494+
cc_library("${TARGET_NAME}_proto" SRCS "${grpc_proto_srcs}")
495+
496+
add_custom_command(
497+
OUTPUT "${grpc_grpc_srcs}" "${grpc_grpc_hdrs}"
498+
COMMAND ${PROTOBUF_PROTOC_EXECUTABLE}
499+
ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}" -I "${PROTO_PATH}"
500+
--plugin=protoc-gen-grpc="${GRPC_CPP_PLUGIN}" "${ABS_PROTO}"
501+
DEPENDS "${ABS_PROTO}" ${PROTOBUF_PROTOC_EXECUTABLE} extern_grpc)
502+
503+
# FIXME(typhoonzero): grpc generated code do not generate virtual-dtor, mark it
504+
# as compiler warnings instead of error. Should try remove the warnings also.
505+
set_source_files_properties(
506+
${grpc_grpc_srcs}
507+
PROPERTIES
508+
COMPILE_FLAGS "-Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor")
509+
cc_library("${TARGET_NAME}_grpc" SRCS "${grpc_grpc_srcs}")
510+
511+
set_source_files_properties(
512+
${grpc_library_SRCS}
513+
PROPERTIES
514+
COMPILE_FLAGS "-Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor")
515+
cc_library("${TARGET_NAME}" SRCS "${grpc_library_SRCS}" DEPS "${TARGET_NAME}_grpc" "${TARGET_NAME}_proto" "${grpc_library_DEPS}")
516+
endfunction()

0 commit comments

Comments
 (0)