Skip to content

Commit 1b269db

Browse files
author
yi.wu
committed
use deb to build paddle-core image
1 parent f66fd44 commit 1b269db

File tree

7 files changed

+130
-35
lines changed

7 files changed

+130
-35
lines changed
Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#!/bin/bash
22
set -e
3+
ARCH=$(uname -i)
34
apt-get update
45
apt-get install -y dh-make
56
cd ~
@@ -8,28 +9,35 @@ mkdir -p ~/dist/cpu
89
mkdir -p ~/dist/cpu-noavx
910
mkdir -p ~/dist/gpu-noavx
1011
cd paddle
11-
mkdir build
12+
13+
# clean build dir and third_party dir cache
14+
rm -rf build third_party
15+
mkdir -p build
1216
cd build
13-
cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=ON
17+
cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=ON -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
1418
make -j `nproc`
19+
# FIXME(typhoonzero): add ARCH gpu noavx flag to CPACK_SYSTEM_NAME. Why -D not affect anything?
1520
cpack -D CPACK_GENERATOR='DEB' ..
1621
mv *.deb ~/dist/cpu
1722

1823
rm -rf *
19-
cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=ON -DCUDNN_ROOT=/usr/
24+
ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so
25+
cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=ON -DCUDNN_ROOT=/usr/ -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
2026
make -j `nproc`
2127
cpack -D CPACK_GENERATOR='DEB' ..
2228
mv *.deb ~/dist/gpu
2329

2430

2531
rm -rf *
26-
cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=OFF
32+
rm -f /usr/lib/libcudnn.so
33+
cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=OFF -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
2734
make -j `nproc`
2835
cpack -D CPACK_GENERATOR='DEB' ..
2936
mv *.deb ~/dist/cpu-noavx
3037

3138
rm -rf *
32-
cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=OFF -DCUDNN_ROOT=/usr/
39+
ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so
40+
cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=OFF -DCUDNN_ROOT=/usr/ -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
3341
make -j `nproc`
3442
cpack -D CPACK_GENERATOR='DEB' ..
3543
mv *.deb ~/dist/gpu-noavx

paddle/scripts/docker/buildall.sh

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,31 @@
11
#!/bin/bash
22

3-
BINARIES_DIR=paddle/scripts/docker/buildimage/binaries
43
BUILD_DIR=$PWD/build
4+
DEB_DIST_DIR=$PWD/dist
5+
VERSION=latest
56

67
function build_in_docker() {
78
if [ ! -d $BUILD_DIR ]; then
89
mkdir -p $BUILD_DIR
910
fi
11+
if [ ! -d $DEB_DIST_DIR ]; then
12+
mkdir -p $DEB_DIST_DIR
13+
fi
1014
docker build . -t paddle-build-env -f paddle/scripts/docker/paddle-dev/Dockerfile
1115
# FIXME: need to wait a signal not sleeping
12-
BUILDER=$(docker run -d -v ${PWD}:/paddle paddle-build-env sleep 3600)
13-
# TODO(typhoonzero):
14-
docker exec $BUILDER /bin/bash -c "export BUILD_AND_INSTALL=ON && /paddle/paddle/scripts/docker/build.sh"
15-
mkdir -p $BINARIES_DIR
16-
# docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_pserver_main $BINARIES_DIR
17-
# docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_trainer $BINARIES_DIR
18-
# docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_merge_model $BINARIES_DIR
19-
# docker cp $BUILDER:/usr/local/bin/paddle $BINARIES_DIR
20-
# docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_usage $BINARIES_DIR
21-
#
22-
# docker cp $BUILDER:/usr/local/opt/paddle/share/wheels $BINARIES_DIR
16+
BUILDER=$(docker run -d -v ${PWD}:/root/paddle -v ${DEB_DIST_DIR}:/root/dist paddle-build-env sleep 3600)
17+
# NOTICE: build deb files for real paddle image
18+
docker exec $BUILDER /bin/bash -c "/root/paddle/paddle/scripts/deb/build_scripts/build.sh"
2319

2420
docker stop $BUILDER && docker rm $BUILDER
2521
}
2622

2723
function build_paddle_core() {
28-
docker build . -t paddle-core -f paddle/scripts/docker/paddle-core/Dockerfile
24+
docker build . -t paddle-core:$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile
25+
docker build . -t paddle-core:gpu-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.gpu
26+
docker build . -t paddle-core:cpu-noavx-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.noavx
27+
docker build . -t paddle-core:gpu-noavx-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx
2928
}
3029

3130
build_in_docker
32-
#build_paddle_core
31+
build_paddle_core

paddle/scripts/docker/paddle-core/Dockerfile

Lines changed: 3 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -23,20 +23,10 @@ RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/
2323
pip install --upgrade pip && \
2424
pip install -U 'protobuf==3.1.0'
2525
RUN pip install numpy
26+
# Use different deb file when building different type of images
27+
ADD dist/cpu/*.deb /usr/local/opt/paddle/deb/cpu/
28+
RUN dpkg --force-all -i /usr/local/opt/paddle/deb/cpu/*.deb && rm -f /usr/local/opt/paddle/deb/cpu/*.deb
2629

27-
ADD build/paddle/trainer/paddle_trainer \
28-
build/paddle/pserver/paddle_pserver_main \
29-
build/paddle/trainer/paddle_merge_model \
30-
/usr/local/opt/paddle/bin/
31-
ADD build/paddle/scripts/usage.sh /usr/local/opt/paddle/bin/paddle_usage
32-
ADD paddle/scripts/submit_local.sh.in /usr/bin/paddle
33-
34-
ADD paddle/dist/py_paddle*linux*.whl build/python/dist/paddle*.whl \
35-
/usr/local/opt/paddle/share/wheels/
36-
RUN pip install /usr/local/opt/paddle/share/wheels/py_paddle*linux*.whl && \
37-
pip install /usr/local/opt/paddle/share/wheels/paddle*.whl && \
38-
rm /usr/local/opt/paddle/share/wheels/py_paddle*linux*.whl && \
39-
rm /usr/local/opt/paddle/share/wheels/paddle*.whl
4030
ENV PATH="/usr/local/opt/paddle/bin/:${PATH}"
4131
# default command shows the paddle version and exit
4232
CMD ["paddle", "version"]
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
FROM nvidia/cuda:7.5-cudnn5-runtime-ubuntu14.04
2+
MAINTAINER PaddlePaddle Authors <[email protected]>
3+
4+
# ENV variables
5+
ARG WITH_AVX
6+
ARG WITH_DOC
7+
ARG WITH_STYLE_CHECK
8+
9+
ENV WITH_GPU=OFF
10+
ENV WITH_AVX=${WITH_AVX:-ON}
11+
ENV WITH_DOC=${WITH_DOC:-OFF}
12+
ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF}
13+
14+
ENV HOME /root
15+
ENV LANG en_US.UTF-8
16+
17+
# Use Fix locales to en_US.UTF-8
18+
19+
RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \
20+
apt-get update && \
21+
apt-get install -y python python-pip libgfortran3 && \
22+
apt-get clean -y && \
23+
pip install --upgrade pip && \
24+
pip install -U 'protobuf==3.1.0'
25+
RUN pip install numpy
26+
# Use different deb file when building different type of images
27+
ADD dist/gpu/*.deb /usr/local/opt/paddle/deb/gpu/
28+
RUN dpkg --force-all -i /usr/local/opt/paddle/deb/gpu/*.deb && rm -f /usr/local/opt/paddle/deb/gpu/*.deb
29+
30+
ENV PATH="/usr/local/opt/paddle/bin/:${PATH}"
31+
# default command shows the paddle version and exit
32+
CMD ["paddle", "version"]
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
FROM python:2.7.13-slim
2+
MAINTAINER PaddlePaddle Authors <[email protected]>
3+
4+
# ENV variables
5+
ARG WITH_AVX
6+
ARG WITH_DOC
7+
ARG WITH_STYLE_CHECK
8+
9+
ENV WITH_GPU=OFF
10+
ENV WITH_AVX=${WITH_AVX:-ON}
11+
ENV WITH_DOC=${WITH_DOC:-OFF}
12+
ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF}
13+
14+
ENV HOME /root
15+
ENV LANG en_US.UTF-8
16+
17+
# Use Fix locales to en_US.UTF-8
18+
19+
RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \
20+
apt-get update && \
21+
apt-get install -y libgfortran3 && \
22+
apt-get clean -y && \
23+
pip install --upgrade pip && \
24+
pip install -U 'protobuf==3.1.0'
25+
RUN pip install numpy
26+
# Use different deb file when building different type of images
27+
ADD dist/gpu-noavx/*.deb /usr/local/opt/paddle/deb/gpu-noavx/
28+
RUN dpkg --force-all -i /usr/local/opt/paddle/deb/gpu-noavx/*.deb && rm -f /usr/local/opt/paddle/deb/gpu-noavx/*.deb
29+
30+
31+
ENV PATH="/usr/local/opt/paddle/bin/:${PATH}"
32+
# default command shows the paddle version and exit
33+
CMD ["paddle", "version"]
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
FROM nvidia/cuda:7.5-cudnn5-runtime-ubuntu14.04
2+
MAINTAINER PaddlePaddle Authors <[email protected]>
3+
4+
# ENV variables
5+
ARG WITH_AVX
6+
ARG WITH_DOC
7+
ARG WITH_STYLE_CHECK
8+
9+
ENV WITH_GPU=OFF
10+
ENV WITH_AVX=${WITH_AVX:-ON}
11+
ENV WITH_DOC=${WITH_DOC:-OFF}
12+
ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF}
13+
14+
ENV HOME /root
15+
ENV LANG en_US.UTF-8
16+
17+
# Use Fix locales to en_US.UTF-8
18+
19+
RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \
20+
apt-get update && \
21+
apt-get install -y python python-pip libgfortran3 && \
22+
apt-get clean -y && \
23+
pip install --upgrade pip && \
24+
pip install -U 'protobuf==3.1.0'
25+
RUN pip install numpy
26+
# Use different deb file when building different type of images
27+
ADD dist/cpu-noavx/*.deb /usr/local/opt/paddle/deb/cpu-noavx/
28+
RUN dpkg --force-all -i /usr/local/opt/paddle/deb/cpu-noavx/*.deb && rm -f /usr/local/opt/paddle/deb/cpu-noavx/*.deb
29+
30+
ENV PATH="/usr/local/opt/paddle/bin/:${PATH}"
31+
# default command shows the paddle version and exit
32+
CMD ["paddle", "version"]

paddle/scripts/docker/paddle-dev/Dockerfile

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# A image for building paddle binaries
2-
FROM ubuntu:14.04
2+
# Use cuda devel base image for both cpu and gpu environment
3+
FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04
34
MAINTAINER PaddlePaddle Authors <[email protected]>
45

56
ARG DEBIAN_FRONTEND=noninteractive
@@ -22,8 +23,7 @@ ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF}
2223

2324
ENV HOME /root
2425

25-
RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \
26-
apt-get update && \
26+
RUN apt-get update && \
2727
apt-get install -y git python-pip python-dev openssh-server bison && \
2828
apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \
2929
apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \
@@ -48,5 +48,6 @@ RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \
4848
cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \
4949
cd .. && rm -rf cmake-3.4.1
5050

51+
RUN apt-get install -y swig
5152
# FIXME: wait a long time is OK
5253
CMD ["sleep", "3600"]

0 commit comments

Comments
 (0)