Skip to content

Commit d5ee344

Browse files
hephaexhephaex
committed
python-espnet (Py37, CUDA11.0)
1 parent c82bc44 commit d5ee344

File tree

5 files changed

+377
-0
lines changed

5 files changed

+377
-0
lines changed

‎python-espnet/Dockerfile

Lines changed: 325 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,325 @@
1+
FROM ubuntu:18.04
2+
LABEL maintainer "Mario Cho <[email protected]>"
3+
4+
ENV DEBIAN_FRONTEND=noninteractive \
5+
MPLBACKEND=Svg \
6+
PYTHONUNBUFFERED=1 \
7+
LIBRARY_PATH=/usr/local/cuda/lib64/stubs \
8+
_CUDA_COMPAT_PATH="/usr/local/cuda/compat" \
9+
LD_LIBRARY_PATH="/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/compat/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/include/x86_64-linux-gnu"\
10+
PATH="/usr/local/nvidia/bin:/usr/local/cuda/bin:/minicondaenv/espnet/bin:/usr/local/sbin:/usr/bin/cmake/bin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/tensorrt/bin" \
11+
LANG=C.UTF-8
12+
13+
ENV CUDA_VERSION 11.0.3
14+
ENV CUDA_PKG_VERSION 11-0=$CUDA_VERSION-1
15+
ENV NCCL_VERSION 2.8.4
16+
ENV CUDNN_VERSION 8.0.5.39
17+
LABEL com.nvidia.cudnn.version="${CUDNN_VERSION}"
18+
19+
# nvidia-container-runtime
20+
ENV NVIDIA_VISIBLE_DEVICES all
21+
ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
22+
ENV NVIDIA_REQUIRE_CUDA "cuda>=11.0 brand=tesla,driver>=418,driver<419 brand=tesla,driver>=440,driver<441 brand=tesla,driver>=450,driver<451"
23+
ENV CUDA_HOME /usr/local/cuda
24+
25+
RUN apt-get update && \
26+
apt-get -y install --no-install-recommends \
27+
fonts-nanum \
28+
fonts-nanum-coding \
29+
fonts-nanum-extra \
30+
libasound-dev \
31+
libcairo2-dev \
32+
libgif-dev \
33+
libjpeg-dev \
34+
liblapack-dev \
35+
libnuma-dev \
36+
libopenblas-dev \
37+
libpmi2-0-dev \
38+
nano \
39+
numactl \
40+
vim \
41+
openssh-client openssh-server \
42+
apt-utils \
43+
autoconf \
44+
automake \
45+
bc \
46+
build-essential \
47+
ca-certificates \
48+
cmake \
49+
curl \
50+
debhelper dh-virtualenv \
51+
ffmpeg \
52+
flac \
53+
gawk \
54+
gfortran \
55+
git \
56+
libatlas-base-dev \
57+
libatlas3-base libtool \
58+
libsndfile1-dev \
59+
libssl-dev \
60+
libtool \
61+
python2.7 \
62+
python3 \
63+
sox \
64+
subversion \
65+
unzip \
66+
wget \
67+
zip \
68+
zlib1g-dev \
69+
&& \
70+
apt-get clean && \
71+
rm -rf /var/lib/apt/lists/*
72+
73+
## FROM CUDA 11.0 base [https://gitlab.com/nvidia/cuda/blob/ubuntu18.04/11.0/base/Dockerfile]
74+
75+
RUN apt-get update && \
76+
apt-get install -y --no-install-recommends gnupg2 curl ca-certificates && \
77+
curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub | apt-key add - && \
78+
echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 /" > /etc/apt/sources.list.d/cuda.list && \
79+
echo "deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 /" > /etc/apt/sources.list.d/nvidia-ml.list && \
80+
rm -rf /var/lib/apt/lists/*
81+
82+
# For libraries in the cuda-compat-* package: https://docs.nvidia.com/cuda/eula/index.html#attachment-a
83+
RUN apt-get update && apt-get install -y --no-install-recommends \
84+
cuda-cudart-11-0=11.0.221-1 \
85+
cuda-compat-11-0 && \
86+
ln -s cuda-11.0 /usr/local/cuda && \
87+
rm -rf /var/lib/apt/lists/*
88+
89+
# Required for nvidia-docker v1
90+
RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf && \
91+
echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf
92+
93+
## FROM CUDA 11.0 runtime [https://gitlab.com/nvidia/cuda/blob/ubuntu18.04/11.0/runtime/Dockerfile]
94+
95+
RUN apt-get update && apt-get install -y --no-install-recommends \
96+
cuda-libraries-11-0=11.0.3-1 \
97+
libnpp-11-0=11.1.0.245-1 \
98+
cuda-nvtx-11-0=11.0.167-1 \
99+
libcublas-11-0=11.2.0.252-1 \
100+
libcusparse-11-0=11.1.1.245-1 \
101+
libnccl2=$NCCL_VERSION-1+cuda11.0 \
102+
&& \
103+
apt-mark hold libcublas-11-0 libnccl2 && \
104+
rm -rf /var/lib/apt/lists/*
105+
106+
## FROM CUDA 11.0 devel [https://gitlab.com/nvidia/cuda/blob/ubuntu18.04/11.0/devel/Dockerfile]
107+
108+
RUN apt-get update && apt-get install -y --no-install-recommends \
109+
cuda-cudart-dev-11-0=11.0.221-1 \
110+
cuda-command-line-tools-11-0=11.0.3-1 \
111+
cuda-minimal-build-11-0=11.0.3-1 \
112+
cuda-libraries-dev-11-0=11.0.3-1 \
113+
cuda-nvml-dev-11-0=11.0.167-1 \
114+
libnpp-dev-11-0=11.1.0.245-1 \
115+
libnccl-dev=2.8.4-1+cuda11.0 \
116+
libcublas-dev-11-0=11.2.0.252-1 \
117+
libcusparse-dev-11-0=11.1.1.245-1 \
118+
&& \
119+
apt-mark hold libcublas-dev-11-0 libnccl-dev && \
120+
rm -rf /var/lib/apt/lists/*
121+
122+
## FROM CUDA 11.0-CUDNN 8 devel
123+
124+
RUN apt-get update && apt-get install -y --no-install-recommends \
125+
libcudnn8=$CUDNN_VERSION-1+cuda11.0 \
126+
libcudnn8-dev=$CUDNN_VERSION-1+cuda11.0 && \
127+
apt-mark hold libcudnn8 && \
128+
rm -rf /var/lib/apt/lists/*
129+
130+
131+
RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py37_4.9.2-Linux-x86_64.sh -O miniconda.sh && \
132+
mkdir -p /opt && \
133+
sh miniconda.sh -b -p /opt/conda && \
134+
rm miniconda.sh && \
135+
ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
136+
echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \
137+
echo "conda activate base" >> ~/.bashrc && \
138+
find /opt/conda/ -follow -type f -name '*.a' -delete && \
139+
find /opt/conda/ -follow -type f -name '*.js.map' -delete && \
140+
/opt/conda/bin/conda clean -afy
141+
142+
# Install Kaldi
143+
#RUN git clone https://github.com/kaldi-asr/kaldi
144+
145+
WORKDIR /tmp
146+
# OpenBlas
147+
RUN git clone -q --branch=master git://github.com/xianyi/OpenBLAS.git && \
148+
cd OpenBLAS && \
149+
make DYNAMIC_ARCH=1 NO_AFFINITY=1 NUM_THREADS=48 FC=gfortran && \
150+
make install
151+
152+
# Install Open UCX
153+
WORKDIR /tmp
154+
RUN wget https://github.com/openucx/ucx/archive/v1.9.0.tar.gz && \
155+
tar xvf v1.9.0.tar.gz && \
156+
cd ucx-1.9.0 && \
157+
bash ./autogen.sh && \
158+
mkdir build && \
159+
cd build && \
160+
../configure --prefix=/usr/local/ucx && \
161+
make -j$(nproc) && \
162+
make install
163+
164+
# Install Open MPI
165+
RUN mkdir /tmp/openmpi && \
166+
cd /tmp/openmpi && \
167+
wget https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.0.tar.gz && \
168+
tar zxf openmpi-4.1.0.tar.gz && \
169+
cd openmpi-4.1.0 && \
170+
./configure --enable-orterun-prefix-by-default --with-cuda --with-ucx=/usr/local/ucx --enable-mca-no-build=btl-uct && \
171+
make -j $(nproc) all && \
172+
make install && \
173+
ldconfig && \
174+
rm -rf /tmp/openmpi*
175+
176+
# Install OpenSSH for MPI to communicate between containers
177+
RUN apt-get install -y --no-install-recommends openssh-client openssh-server && \
178+
mkdir -p /var/run/sshd
179+
180+
# Allow OpenSSH to talk to containers without asking for confirmation
181+
RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new && \
182+
echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \
183+
mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
184+
185+
# Create a wrapper for OpenMPI to allow running as root by default
186+
RUN mv /usr/local/bin/mpirun /usr/local/bin/mpirun.real && \
187+
echo '#!/bin/bash' > /usr/local/bin/mpirun && \
188+
echo 'mpirun.real --allow-run-as-root "$@"' >> /usr/local/bin/mpirun && \
189+
chmod a+x /usr/local/bin/mpirun
190+
191+
# Configure OpenMPI to run good defaults:
192+
RUN echo "btl_tcp_if_exclude = lo,docker0" >> /usr/local/etc/openmpi-mca-params.conf
193+
194+
# Install OpenSSH for MPI to communicate between containers
195+
RUN mkdir -p /var/run/sshd
196+
197+
# Allow OpenSSH to talk to containers without asking for confirmation
198+
RUN cat /etc/ssh/ssh_config | grep -v StrictHostKeyChecking > /etc/ssh/ssh_config.new && \
199+
echo " StrictHostKeyChecking no" >> /etc/ssh/ssh_config.new && \
200+
mv /etc/ssh/ssh_config.new /etc/ssh/ssh_config
201+
202+
# Install git-lfs
203+
WORKDIR /tmp
204+
RUN curl -sLO https://github.com/git-lfs/git-lfs/releases/download/v2.13.1/git-lfs-linux-amd64-v2.13.1.tar.gz && \
205+
tar -zxf git-lfs-linux-amd64-v2.13.1.tar.gz && \
206+
bash install.sh && \
207+
rm -rf /tmp/*
208+
209+
# Install VSCode
210+
RUN curl -fL https://github.com/cdr/code-server/releases/download/v3.8.0/code-server-3.8.0-linux-amd64.tar.gz \
211+
| tar -C /usr/local/lib -xz && \
212+
mv /usr/local/lib/code-server-3.8.0-linux-amd64 /usr/local/lib/code-server-3.8.0 && \
213+
ln -s /usr/local/lib/code-server-3.8.0/bin/code-server /usr/local/bin/code-server
214+
215+
RUN git clone https://github.com/hephaex/kaldi.git /opt/kaldi && \
216+
cd /opt/kaldi/tools && \
217+
./extras/install_mkl.sh -sp debian intel-mkl-64bit-2019.2-057 && \
218+
apt-get clean && \
219+
rm -rf /var/lib/apt/lists/* && \
220+
make -j $(nproc) all && \
221+
rm -r openfst-*/src && \
222+
./extras/install_beamformit.sh && \
223+
./extras/install_irstlm.sh && \
224+
cd /opt/kaldi/src && \
225+
./configure --shared --use-cuda \
226+
--cudatk-dir=/usr/local/cuda/ \
227+
--mathlib=ATLAS \
228+
--cuda-arch="-gencode arch=compute_52,code=sm_52 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80" && \
229+
make depend -j $(nproc) && \
230+
make -j $(nproc) && \
231+
find /opt/kaldi/src -name "*.o" -exec rm -f {} \; && \
232+
find /opt/kaldi/src -name "*.o" -exec rm -f {} \;
233+
234+
235+
ENV CUDA_VER 11.0
236+
#WORKDIR /
237+
238+
RUN git clone https://github.com/espnet/espnet /opt/espnet
239+
# IF using a local ESPNet repository, a temporary file containing the ESPnet git repo is copied over
240+
#ARG ESPNET_ARCHIVE=./espnet-local.tar
241+
#COPY ${ESPNET_ARCHIVE} /espnet-local.tar
242+
243+
244+
# Download ESPnet
245+
#RUN echo "Getting ESPnet sources from local repository, in temporary file: " ${ESPNET_ARCHIVE}
246+
#RUN mkdir /espnet
247+
#RUN tar xf espnet-local.tar -C /espnet/
248+
249+
#RUN cd espnet && \
250+
# rm -rf docker egs test utils
251+
252+
# Install espnet
253+
WORKDIR /opt/espnet/tools
254+
ENV TH_VERSION 1.7.1
255+
ENV CUDA_VER 11.0
256+
# Replace nvidia-smi for nvcc because docker does not load nvidia-smi
257+
RUN if [ -z "$( which nvcc )" ]; then \
258+
echo "Build without CUDA" && \
259+
MY_OPTS='CUPY_VERSION="" TH_VERSION=1.7.1'; \
260+
else \
261+
echo "Build with CUDA" && \
262+
# Docker containers cannot load cuda libs during build.
263+
# So, their checks on cuda packages are disabled.
264+
sed -i '200s|install.py|install.py --no-cuda --no-cupy |' Makefile && \
265+
export CFLAGS="-I${CUDA_HOME}/include ${CFLAGS}" && \
266+
MY_OPTS="CUDA_VERSION=${CUDA_VER}" && \
267+
. ./setup_cuda_env.sh /usr/local/cuda; \
268+
fi; \
269+
if [ "${CUDA_VER}" = "11.0" ]; then \
270+
# warpctc is not supported from Pytorch 1.3.1
271+
MY_OPTS="${MY_OPTS} TH_VERSION=1.7.1"; \
272+
fi; \
273+
echo "Make with options ${MY_OPTS}" && \
274+
ln -s /opt/kaldi ./ && \
275+
./setup_python.sh /opt/conda/bin/python3 && \
276+
make KALDI=/opt/kaldi ${MY_OPTS}
277+
278+
# Add extra libraries (VC/TTS)
279+
280+
WORKDIR /opt/espnet/tools
281+
RUN /opt/conda/bin/python3 -m pip install --no-cache-dir \
282+
torch==1.7.1 parallel_wavegan && \
283+
/opt/conda/bin/python3 -m pip install --no-cache-dir \
284+
torch==1.7.1 git+https://github.com/cybertronai/pytorch-lamb
285+
286+
RUN /opt/conda/bin/python3 -m pip install --no-cache-dir \
287+
git+https://github.com/lanpa/tensorboardX \
288+
tornado==6.0.4 \
289+
pystan==2.19.1.1 \
290+
pycairo==1.19.0 \
291+
jupyter==1.0.0 \
292+
python-language-server[all]
293+
294+
COPY ./service-defs /etc/backend.ai/service-defs
295+
RUN echo ". /opt/espnet/tools/activate_python.sh" >> /etc/profile
296+
297+
RUN \
298+
# mkdir -p /opt/espnet/egs/ljspeech/tts1/downloads && \
299+
# cd /opt/espnet/egs/ljspeech/tts1/downloads && \
300+
# wget http://data.keithito.com/data/speech/LJSpeech-1.1.tar.bz2 && \
301+
# tar -vxf ./*.tar.bz2 && \
302+
# rm ./*.tar.bz2 && \
303+
chmod +x /opt/espnet/tools/activate_python.sh && \
304+
chmod -R 777 /opt/espnet
305+
# chmod -R 777 /opt/espnet/egs/ljspeech/tts1/downloads
306+
307+
# Install ipython kernelspec
308+
Run /opt/conda/bin/python3 -m ipykernel install --display-name "A4003 (ESPNet, PyTorch 1.7.1 on Python 3.7 & CUDA 11.0)" && \
309+
cat /usr/local/share/jupyter/kernels/python3/kernel.json
310+
311+
# Backend.AI specifics
312+
LABEL ai.backend.kernelspec="1" \
313+
ai.backend.envs.corecount="OPENBLAS_NUM_THREADS,OMP_NUM_THREADS,NPROC" \
314+
ai.backend.features="batch query uid-match user-input" \
315+
ai.backend.base-distro="ubuntu16.04" \
316+
ai.backend.resource.min.cpu="1" \
317+
ai.backend.resource.min.mem="1g" \
318+
ai.backend.resource.min.cuda.device=0 \
319+
ai.backend.resource.min.cuda.shares=0 \
320+
ai.backend.runtime-type="python" \
321+
ai.backend.runtime-path="/opt/conda/bin/python3" \
322+
ai.backend.service-ports="ipython:pty:3000,jupyter:http:8081,tensorboard:http:6006"
323+
324+
WORKDIR /home/work
325+
# vim: ft=dockerfile

‎python-espnet/service-defs/ipython.json

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"command": [
3+
"{runtime_path}", "-m", "IPython"
4+
]
5+
}

‎python-espnet/service-defs/jupyter.json

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"prestart": [
3+
{
4+
"action": "write_tempfile",
5+
"args": {
6+
"body": [
7+
"c.NotebookApp.allow_root = True\n",
8+
"c.NotebookApp.ip = \"0.0.0.0\"\n",
9+
"c.NotebookApp.port = {ports[0]}\n",
10+
"c.NotebookApp.token = \"\"\n",
11+
"c.FileContentsManager.delete_to_trash = False\n"
12+
]
13+
},
14+
"ref": "jupyter_cfg"
15+
}
16+
],
17+
"command": [
18+
"{runtime_path}", "-m", "notebook", "--no-browser", "--config", "{jupyter_cfg}"
19+
]
20+
}

‎python-espnet/service-defs/sftp.json

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"command": [
3+
"{runtime_path}", "-m", "sftpserver", "--port", "{ports[0]}"
4+
]
5+
}

‎python-espnet/service-defs/tensorboard.json

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
{
2+
"prestart": [
3+
{
4+
"action": "mkdir",
5+
"args": {
6+
"path": "/home/work/logs"
7+
}
8+
}
9+
],
10+
"command": [
11+
"{runtime_path}",
12+
"-m", "tensorboard.main",
13+
"--port", "{ports[0]}",
14+
"--bind_all"
15+
],
16+
"allowed_arguments": [
17+
"--logdir"
18+
],
19+
"default_arguments": {
20+
"--logdir": "/home/work/logs"
21+
}
22+
}

0 commit comments

Comments
 (0)