Skip to content

Commit 08a8c1d

Browse files
JiabinYangJiabinYang
authored
Cherry pick/update protobuf (#15793)
* test=develop, change md5 for patchELF * test=develop, update protobuf * test=develop, update protobuf in Dockerfile used by CI * test=develop, install requirements before start for Linux * test=develop, uninstall protobuf on linux brefore install latest version of it * test=develop, fix pip * test=develop, install protobuf in linux * test=develop, fix patch ELF install failed * test=develop, change md5 for patchELF * test=release/1.3, cherry-pick
1 parent cbec20e commit 08a8c1d

File tree

6 files changed

+31
-15
lines changed

6 files changed

+31
-15
lines changed

‎cmake/external/protobuf.cmake

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,7 @@ FUNCTION(build_protobuf TARGET_NAME BUILD_FOR_HOST)
203203
ENDIF()
204204

205205
SET(PROTOBUF_REPO "https://github.com/google/protobuf.git")
206-
SET(PROTOBUF_TAG "9f75c5aa851cd877fb0d93ccc31b8567a6706546")
206+
SET(PROTOBUF_TAG "v3.6.1")
207207

208208
ExternalProject_Add(
209209
${TARGET_NAME}
@@ -231,7 +231,7 @@ FUNCTION(build_protobuf TARGET_NAME BUILD_FOR_HOST)
231231
)
232232
ENDFUNCTION()
233233

234-
SET(PROTOBUF_VERSION 3.1)
234+
SET(PROTOBUF_VERSION 3.6.1)
235235

236236
IF(NOT PROTOBUF_FOUND)
237237
build_protobuf(extern_protobuf FALSE)

‎cmake/external/python.cmake

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,8 +74,8 @@ IF(PYTHONINTERP_FOUND)
7474
find_python_module(wheel REQUIRED)
7575
find_python_module(google.protobuf REQUIRED)
7676
FIND_PACKAGE(NumPy REQUIRED)
77-
IF(${PY_GOOGLE.PROTOBUF_VERSION} AND ${PY_GOOGLE.PROTOBUF_VERSION} VERSION_LESS "3.0.0")
78-
MESSAGE(FATAL_ERROR "Found Python Protobuf ${PY_GOOGLE.PROTOBUF_VERSION} < 3.0.0, "
77+
IF(${PY_GOOGLE.PROTOBUF_VERSION} AND ${PY_GOOGLE.PROTOBUF_VERSION} VERSION_LESS "3.6.1")
78+
MESSAGE(FATAL_ERROR "Found Python Protobuf ${PY_GOOGLE.PROTOBUF_VERSION} < 3.6.1, "
7979
"please use pip to upgrade protobuf. pip install -U protobuf")
8080
ENDIF()
8181
ENDIF(PYTHONINTERP_FOUND)

‎paddle/scripts/paddle_build.sh

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ function cmake_gen() {
8888
-DPYTHON_INCLUDE_DIR:PATH=/Library/Frameworks/Python.framework/Versions/3.5/include/python3.5m/
8989
-DPYTHON_LIBRARY:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.5/lib/libpython3.5m.dylib"
9090
WITH_FLUID_ONLY=${WITH_FLUID_ONLY:-ON}
91+
pip3.5 uninstall -y protobuf
9192
pip3.5 install --user -r ${PADDLE_ROOT}/python/requirements.txt
9293
else
9394
exit 1
@@ -101,6 +102,7 @@ function cmake_gen() {
101102
-DPYTHON_INCLUDE_DIR:PATH=/Library/Frameworks/Python.framework/Versions/3.6/include/python3.6m/
102103
-DPYTHON_LIBRARY:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.6/lib/libpython3.6m.dylib"
103104
WITH_FLUID_ONLY=${WITH_FLUID_ONLY:-ON}
105+
pip3.6 uninstall -y protobuf
104106
pip3.6 install --user -r ${PADDLE_ROOT}/python/requirements.txt
105107
else
106108
exit 1
@@ -114,6 +116,7 @@ function cmake_gen() {
114116
-DPYTHON_INCLUDE_DIR:PATH=/Library/Frameworks/Python.framework/Versions/3.7/include/python3.7m/
115117
-DPYTHON_LIBRARY:FILEPATH=/Library/Frameworks/Python.framework/Versions/3.7/lib/libpython3.7m.dylib"
116118
WITH_FLUID_ONLY=${WITH_FLUID_ONLY:-ON}
119+
pip3.7 uninstall -y protobuf
117120
pip3.7 install --user -r ${PADDLE_ROOT}/python/requirements.txt
118121
else
119122
exit 1
@@ -128,31 +131,44 @@ function cmake_gen() {
128131
PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/python/cp27-cp27m/bin/python
129132
-DPYTHON_INCLUDE_DIR:PATH=/opt/python/cp27-cp27m/include/python2.7
130133
-DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-2.7.11-ucs2/lib/libpython2.7.so"
134+
pip uninstall -y protobuf
135+
pip install -r ${PADDLE_ROOT}/python/requirements.txt
131136
elif [ "$1" == "cp27-cp27mu" ]; then
132137
export LD_LIBRARY_PATH=/opt/_internal/cpython-2.7.11-ucs4/lib:${LD_LIBRARY_PATH#/opt/_internal/cpython-2.7.11-ucs2/lib:}
133138
export PATH=/opt/python/cp27-cp27mu/bin/:${PATH}
134139
PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/python/cp27-cp27mu/bin/python
135140
-DPYTHON_INCLUDE_DIR:PATH=/opt/python/cp27-cp27mu/include/python2.7
136141
-DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-2.7.11-ucs4/lib/libpython2.7.so"
142+
pip uninstall -y protobuf
143+
pip install -r ${PADDLE_ROOT}/python/requirements.txt
137144
elif [ "$1" == "cp35-cp35m" ]; then
138145
export LD_LIBRARY_PATH=/opt/_internal/cpython-3.5.1/lib/:${LD_LIBRARY_PATH}
139146
export PATH=/opt/_internal/cpython-3.5.1/bin/:${PATH}
140147
export PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/_internal/cpython-3.5.1/bin/python3
141148
-DPYTHON_INCLUDE_DIR:PATH=/opt/_internal/cpython-3.5.1/include/python3.5m
142149
-DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-3.5.1/lib/libpython3.so"
150+
pip3.5 uninstall -y protobuf
151+
pip3.5 install -r ${PADDLE_ROOT}/python/requirements.txt
143152
elif [ "$1" == "cp36-cp36m" ]; then
144153
export LD_LIBRARY_PATH=/opt/_internal/cpython-3.6.0/lib/:${LD_LIBRARY_PATH}
145154
export PATH=/opt/_internal/cpython-3.6.0/bin/:${PATH}
146155
export PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/_internal/cpython-3.6.0/bin/python3
147156
-DPYTHON_INCLUDE_DIR:PATH=/opt/_internal/cpython-3.6.0/include/python3.6m
148157
-DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-3.6.0/lib/libpython3.so"
158+
pip3.6 uninstall -y protobuf
159+
pip3.6 install -r ${PADDLE_ROOT}/python/requirements.txt
149160
elif [ "$1" == "cp37-cp37m" ]; then
150161
export LD_LIBRARY_PATH=/opt/_internal/cpython-3.7.0/lib/:${LD_LIBRARY_PATH}
151162
export PATH=/opt/_internal/cpython-3.7.0/bin/:${PATH}
152163
export PYTHON_FLAGS="-DPYTHON_EXECUTABLE:FILEPATH=/opt/_internal/cpython-3.7.0/bin/python3.7
153164
-DPYTHON_INCLUDE_DIR:PATH=/opt/_internal/cpython-3.7.0/include/python3.7m
154165
-DPYTHON_LIBRARIES:FILEPATH=/opt/_internal/cpython-3.7.0/lib/libpython3.so"
166+
pip3.7 uninstall -y protobuf
167+
pip3.7 install -r ${PADDLE_ROOT}/python/requirements.txt
155168
fi
169+
else
170+
pip uninstall -y protobuf
171+
pip install -r ${PADDLE_ROOT}/python/requirements.txt
156172
fi
157173
fi
158174

‎python/requirements.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
requests==2.9.2
22
numpy>=1.12
3-
protobuf==3.1
3+
protobuf>=3.6
44
recordio>=0.1.0
55
matplotlib==2.2.3 # TODO: let python3 paddlepaddle package use latest matplotlib
66
rarfile

‎tools/manylinux1/Dockerfile.x64

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,10 @@ RUN wget --no-check-certificate -qO- https://storage.googleapis.com/golang/go1.8
3131
ENV GOROOT=/usr/local/go GOPATH=/root/gopath
3232
ENV PATH=${GOROOT}/bin:${GOPATH}/bin:${PATH}
3333

34-
# protobuf 3.1.0
35-
RUN cd /opt && wget -q --no-check-certificate https://github.com/google/protobuf/releases/download/v3.1.0/protobuf-cpp-3.1.0.tar.gz && \
36-
tar xzf protobuf-cpp-3.1.0.tar.gz && \
37-
cd protobuf-3.1.0 && ./configure && make -j4 && make install && cd .. && rm -f protobuf-cpp-3.1.0.tar.gz
34+
# protobuf 3.6.1
35+
RUN cd /opt && wget -q --no-check-certificate https://github.com/google/protobuf/releases/download/v3.6.1/protobuf-cpp-3.6.1.tar.gz && \
36+
tar xzf protobuf-cpp-3.6.1.tar.gz && \
37+
cd protobuf-3.6.1 && ./configure && make -j4 && make install && cd .. && rm -f protobuf-cpp-3.6.1.tar.gz
3838

3939
RUN wget https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/python/requirements.txt -O /root/requirements.txt
4040

‎tools/manylinux1/build_scripts/build.sh

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ OPENSSL_ROOT=openssl-1.1.0i
1717
OPENSSL_HASH=ebbfc844a8c8cc0ea5dc10b86c9ce97f401837f3fa08c17b2cdadc118253cf99
1818
EPEL_RPM_HASH=e5ed9ecf22d0c4279e92075a64c757ad2b38049bcf5c16c4f2b75d5f6860dc0d
1919
DEVTOOLS_HASH=a8ebeb4bed624700f727179e6ef771dafe47651131a00a78b342251415646acc
20-
PATCHELF_HASH=d9afdff4baeacfbc64861454f368b7f2c15c44d245293f7587bbf726bfe722fb
20+
PATCHELF_HASH=f2aa40a6148cb3b0ca807a1bf836b081793e55ec9e5540a5356d800132be7e0a
2121
CURL_ROOT=curl-7.49.1
2222
CURL_HASH=eb63cec4bef692eab9db459033f409533e6d10e20942f4b060b32819e81885f1
2323
AUTOCONF_ROOT=autoconf-2.69
@@ -107,11 +107,11 @@ curl-config --features
107107
rm -rf /usr/local/ssl
108108

109109
# Install patchelf (latest with unreleased bug fixes)
110-
curl -sLO http://nipy.bic.berkeley.edu/manylinux/patchelf-0.9njs2.tar.gz
111-
check_sha256sum patchelf-0.9njs2.tar.gz $PATCHELF_HASH
112-
tar -xzf patchelf-0.9njs2.tar.gz
113-
(cd patchelf-0.9njs2 && ./configure && make && make install)
114-
rm -rf patchelf-0.9njs2.tar.gz patchelf-0.9njs2
110+
curl -sLO https://nixos.org/releases/patchelf/patchelf-0.9/patchelf-0.9.tar.gz
111+
check_sha256sum patchelf-0.9.tar.gz $PATCHELF_HASH
112+
tar -xzf patchelf-0.9.tar.gz
113+
(cd patchelf-0.9 && ./configure && make && make install)
114+
rm -rf patchelf-0.9.tar.gz patchelf-0.9
115115

116116
# Install latest pypi release of auditwheel
117117
LD_LIBRARY_PATH="${ORIGINAL_LD_LIBRARY_PATH}:$(dirname ${PY35_BIN})/lib" $PY35_BIN/pip install auditwheel

0 commit comments

Comments
 (0)