Skip to content

Commit 3c26ff9

Browse files
hephaexhephaex
committed
add MKL fuctrion for numpy , scipy
1 parent 263bcc4 commit 3c26ff9

File tree

1 file changed

+180
-0
lines changed

1 file changed

+180
-0
lines changed

‎python-tensorflow/Dockerfile.2.2

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
FROM lablup/common-base:20.09
2+
3+
ARG TF_BUILD_VERSION=r2.2
4+
# Install the most recent bazel release.
5+
ENV BAZEL_VERSION 2.0.0
6+
7+
# Set up Bazel.
8+
9+
# Running bazel inside a `docker build` command causes trouble, cf:
10+
# https://github.com/bazelbuild/bazel/issues/134
11+
# The easiest solution is to set up a bazelrc file forcing --batch.
12+
RUN echo "startup --batch" >>/etc/bazel.bazelrc
13+
# Similarly, we need to workaround sandboxing issues:
14+
# https://github.com/bazelbuild/bazel/issues/418
15+
RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \
16+
>>/etc/bazel.bazelrc
17+
18+
WORKDIR /
19+
RUN mkdir /bazel && \
20+
cd /bazel && \
21+
curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \
22+
curl -H "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36" -fSsL -o /bazel/LICENSE.txt https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE && \
23+
chmod +x bazel-*.sh && \
24+
./bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \
25+
cd / && \
26+
rm -f /bazel/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh
27+
28+
29+
RUN python3 -m pip install --no-cache-dir --upgrade \
30+
-q git+https://github.com/google-research/tensorflow_constrained_optimization
31+
32+
RUN python3 -m pip install --no-cache-dir -I \
33+
tensorflow-transform==0.22.0 \
34+
tensorflow_model_analysis==0.22.2 \
35+
tensorflow-metadata==0.22.2 \
36+
tensorflow-datasets==3.2.0 \
37+
tensorflow_probability==0.10.1 \
38+
tensorflow-addons==0.8.3 \
39+
neural-structured-learning==1.1.0 \
40+
tensorflow-model-optimization==0.3.0 \
41+
tfx==0.22.0 \
42+
tfx-bsl==0.22.1 \
43+
tf-agents==0.5.0 \
44+
tensorflow-hub==0.8.0 \
45+
tensorflow_text==2.2.1 \
46+
tensorflow-graphics-gpu==1.0.0 \
47+
tensorflow-gan==2.0.0 \
48+
tensorflow-data-validation==0.22.2 \
49+
tensorflow-plot==0.3.2 \
50+
tensorflow-lattice==2.0.5 \
51+
tensorflow_ranking==0.3.1 \
52+
tensorflow-compression==1.3 \
53+
mesh-tensorflow==0.1.13 \
54+
tensorflow-io==0.13.0 \
55+
jupyterlab-nvdashboard==0.3.1
56+
57+
RUN python3 -m pip install --no-cache-dir --extra-index-url \
58+
https://developer.download.nvidia.com/compute/redist \
59+
nvidia-dali-cuda110
60+
61+
RUN python3 -m pip uninstall -y \
62+
numpy==1.19.2 \
63+
scipy==1.4.1 \
64+
tensorflow==2.3.1 \
65+
tensorflow-gpu==2.3.1 \
66+
&& \
67+
rm -rf /var/lib/apt/lists/* && \
68+
rm -rf /root/.cache
69+
70+
# Install scipy
71+
ENV SCIPY_VERSION 1.5.2
72+
73+
WORKDIR /tmp
74+
COPY ./numpy-1.18.5-cp36-cp36m-linux_x86_64.whl /tmp
75+
RUN git clone --branch=v${SCIPY_VERSION} --depth=1 https://github.com/scipy/scipy.git scipy && \
76+
cd scipy && \
77+
git checkout -b v${SCIPY_VERSION} && \
78+
cp site.cfg.example site.cfg && \
79+
echo "[mkl]" >> site.cfg && \
80+
echo "include_dirs = /opt/intel/compilers_and_libraries_2020.3.279/linux/mkl/include/" >> site.cfg && \
81+
echo "library_dirs = /opt/intel/compilers_and_libraries_2020.3.279/linux/mkl/lib/intel64" >> site.cfg && \
82+
echo "mkl_libs = mkl_rt" >> site.cfg && \
83+
echo "lapack_libs =" >> site.cfg && \
84+
python3 -m pip install -U --no-cache-dir \
85+
cython \
86+
/tmp/numpy-1.18.5-cp36-cp36m-linux_x86_64.whl && \
87+
python3 setup.py install
88+
89+
90+
# Download and build TensorFlow.
91+
WORKDIR /tensorflow
92+
93+
# Download and build TensorFlow.
94+
# Enable checking out both tags and branches
95+
RUN export TAG_PREFIX="v" && \
96+
echo ${TF_BUILD_VERSION} | grep -q ^${TAG_PREFIX}; \
97+
if [ $? -eq 0 ]; then \
98+
git clone --depth=1 https://github.com/tensorflow/tensorflow.git . && \
99+
git fetch --tags && \
100+
git checkout ${TF_BUILD_VERSION}; \
101+
else \
102+
git clone --depth=1 --branch=${TF_BUILD_VERSION} https://github.com/tensorflow/tensorflow.git . ; \
103+
fi
104+
105+
RUN yes "" | python3 configure.py
106+
RUN cp .bazelrc /root/.bazelrc
107+
108+
ENV CI_BUILD_PYTHON ${PYTHON}
109+
ENV WHL_DIR=/tmp/pip3
110+
# Set bazel build parameters in .bazelrc in parameterized_docker_build.sh
111+
# Use --copt=-march values to get optimized builds appropriate for the hardware
112+
# platform of your choice.
113+
# For ivy-bridge or sandy-bridge
114+
# --copt=-march="avx" \
115+
# For haswell, broadwell, or skylake
116+
# --copt=-march="avx2" \
117+
COPY .bazelrc /root/.mkl.bazelrc
118+
RUN echo "import /root/.mkl.bazelrc" >>/root/.bazelrc
119+
120+
RUN tensorflow/tools/ci_build/builds/configured CPU \
121+
bazel --bazelrc=/root/.bazelrc build \
122+
-c opt \
123+
--copt=-msse4.1 \
124+
--copt=-msse4.2 \
125+
--copt=-mavx \
126+
--copt=-mavx2 \
127+
--copt=-mfma \
128+
--copt=-mfpmath=both \
129+
--copt=-O3 \
130+
--copt=-Wformat \
131+
--copt=-Wformat-security \
132+
--copt=-fstack-protector \
133+
--copt=-fPIC \
134+
--copt=-fpic \
135+
--config=mkl \
136+
--config=monolithic \
137+
--config=gdr \
138+
--config=verbs \
139+
# --config=ngraph \
140+
--config=numa \
141+
--config=v2 \
142+
--linkopt=-znoexecstack \
143+
--linkopt=-zrelro \
144+
--linkopt=-znow \
145+
--linkopt=-fstack-protector \
146+
--cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \
147+
-k //tensorflow/tools/pip_package:build_pip_package && \
148+
bazel-bin/tensorflow/tools/pip_package/build_pip_package "${WHL_DIR}" && \
149+
python3 -m pip --no-cache-dir install --upgrade "${WHL_DIR}"/tensorflow-*.whl
150+
151+
152+
# Install Horovod, temporarily
153+
RUN HOROVOD_WITH_TENSORFLOW=1 HOROVOD_WITHOUT_PYTORCH=1 HOROVOD_WITHOUT_MXNET=1 \
154+
pip install --no-cache-dir horovod==0.19.5 && \
155+
ldconfig
156+
157+
RUN python3 -m pip install --no-cache-dir \
158+
mpi4py==3.0.3 \
159+
nni==1.8 \
160+
scikit-nni==0.2.1
161+
162+
# Install ipython kernelspec
163+
Run python3 -m ipykernel install --display-name "TensorFlow 2.2 on Python 3.6 & Intel MKL" && \
164+
cat /usr/local/share/jupyter/kernels/python3/kernel.json
165+
166+
# Backend.AI specifics
167+
LABEL ai.backend.kernelspec="1" \
168+
ai.backend.envs.corecount="OPENBLAS_NUM_THREADS,OMP_NUM_THREADS,NPROC" \
169+
ai.backend.features="batch query uid-match user-input" \
170+
ai.backend.base-distro="ubuntu16.04" \
171+
ai.backend.resource.min.cpu="1" \
172+
ai.backend.resource.min.mem="1g" \
173+
ai.backend.resource.min.cuda.device=0 \
174+
ai.backend.resource.min.cuda.shares=0 \
175+
ai.backend.runtime-type="python" \
176+
ai.backend.runtime-path="/usr/bin/python3" \
177+
ai.backend.service-ports="ipython:pty:3000,jupyter:http:8080,jupyterlab:http:8090,vscode:http:8180,tensorboard:http:6006"
178+
179+
WORKDIR /home/work
180+
# vim: ft=dockerfile

0 commit comments

Comments
 (0)