Skip to content

Commit d7a1860

Browse files
yanzhen1233yangxudong
yanzhen1233
authored and
yangxudong
committed
Build docker (#515)
* Updated Docker image
1 parent eed8283 commit d7a1860

File tree

8 files changed

+220
-10
lines changed

8 files changed

+220
-10
lines changed

‎docker/Dockerfile_tf112‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
#FROM tensorflow/tensorflow:1.12.0
2+
FROM my_tensorflow_base:1.12.0-py2
3+
4+
COPY docker/sources_18.04.list /etc/apt/sources.list
5+
6+
# necessary for later commands to take effect
7+
RUN md5sum /etc/apt/sources.list \
8+
&& apt-get update \
9+
&& apt-get install apt-utils inetutils-ping wget curl telnet vim strace libpq-dev curl libsasl2-dev gcc g++ unzip openjdk-8-jdk -y \
10+
&& apt-get install build-essential cython -y \
11+
&& pip install cython \
12+
&& pip install setuptools_scm
13+
# 检查 Cython 是否安装成功
14+
RUN python -c "import Cython"
15+
RUN pip --version
16+
17+
RUN mkdir /EasyRec
18+
COPY requirements /EasyRec/requirements
19+
COPY requirements.txt /EasyRec/
20+
COPY easy_rec /EasyRec/easy_rec/
21+
COPY setup.cfg /EasyRec/
22+
COPY setup.py /EasyRec/
23+
COPY MANIFEST.in /EasyRec/
24+
COPY README.md /EasyRec/
25+
COPY scripts /EasyRec/scripts
26+
27+
RUN curl "http://easyrec.oss-cn-beijing.aliyuncs.com/tools/odpscmd_public_0.45.0.zip" -o /EasyRec/odpscmd_public.zip
28+
RUN mkdir /usr/local/odps_clt/ && cd /usr/local/odps_clt/ && unzip /EasyRec/odpscmd_public.zip
29+
RUN ln -s /usr/local/odps_clt/bin/odpscmd /usr/local/bin/odpscmd
30+
31+
RUN pip install pystack-debugger idna kafka-python -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
32+
# 升级pip
33+
RUN pip install --upgrade pip setuptools wheel
34+
35+
# 安装 setuptools-rust 和 rustc
36+
RUN pip install setuptools-rust
37+
RUN pip install tensorflow_probability==0.5.0
38+
RUN apt-get update && apt-get install -y rustc
39+
RUN apt-get update && \
40+
apt-get install -y rustc && \
41+
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \
42+
. $HOME/.cargo/env
43+
# 安装 cryptography
44+
RUN pip install cryptography
45+
# 安装基础工具链与依赖项
46+
RUN apt-get update && \
47+
apt-get install -y build-essential libssl-dev libffi-dev python-dev && \
48+
apt-get install -y rustc cargo cmake curl
49+
50+
# 设置国内的 Rust 镜像源
51+
RUN echo '[source.crates-io]\n' > $HOME/.cargo/config
52+
RUN echo 'replace-with = "ustc"' >> $HOME/.cargo/config
53+
RUN echo '[source.ustc]\n' >> $HOME/.cargo/config
54+
RUN echo 'registry = "https://mirrors.ustc.edu.cn/crates.io-index"' >> $HOME/.cargo/config
55+
56+
# 确保 curl 支持 HTTP2
57+
RUN curl -V
58+
59+
# 显示安装好的工具链版本，确保已正确安装
60+
RUN rustc --version
61+
RUN cargo --version
62+
RUN cmake --version
63+
64+
RUN pip install -r /EasyRec/requirements/runtime.txt -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
65+
RUN pip install -r /EasyRec/requirements/extra.txt -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
66+
67+
RUN pip install --user -U https://tfsmoke1.oss-cn-zhangjiakou.aliyuncs.com/tunnel_paiio/common_io/py2/common_io-0.1.0-cp27-cp27mu-linux_x86_64.whl
68+
RUN pip install graphlearn
69+
70+
RUN cd /EasyRec && python setup.py install
71+
RUN rm -rf /EasyRec
72+
RUN python -c "import easy_rec; import pyhive; import datahub; import kafka"
73+
74+
COPY docker/hadoop_env.sh /opt/hadoop_env.sh

‎docker/Dockerfile_tf115‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
FROM datascience-registry.cn-beijing.cr.aliyuncs.com/tensorflow/tensorflow:1.15.5
2+
3+
COPY docker/sources_18.04.list /etc/apt/sources.list
4+
5+
# necessary for later commands to take effect
6+
RUN md5sum /etc/apt/sources.list
7+
8+
RUN apt-get update
9+
RUN apt-get install apt-utils inetutils-ping wget curl telnet vim strace libpq-dev curl libsasl2-dev gcc g++ unzip openjdk-8-jdk -y
10+
11+
RUN mkdir /EasyRec
12+
COPY requirements /EasyRec/requirements
13+
COPY requirements.txt /EasyRec/
14+
COPY easy_rec /EasyRec/easy_rec/
15+
COPY setup.cfg /EasyRec/
16+
COPY setup.py /EasyRec/
17+
COPY MANIFEST.in /EasyRec/
18+
COPY README.md /EasyRec/
19+
COPY scripts /EasyRec/scripts
20+
21+
RUN curl "http://easyrec.oss-cn-beijing.aliyuncs.com/tools/odpscmd_public_0.45.0.zip" -o /EasyRec/odpscmd_public.zip
22+
RUN mkdir /usr/local/odps_clt/ && cd /usr/local/odps_clt/ && unzip /EasyRec/odpscmd_public.zip
23+
RUN ln -s /usr/local/odps_clt/bin/odpscmd /usr/local/bin/odpscmd
24+
RUN pip3 install --upgrade pip
25+
RUN pip3 install pystack-debugger idna kafka-python -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
26+
RUN pip3 install -r /EasyRec/requirements/runtime.txt
27+
RUN pip3 install -r /EasyRec/requirements/extra.txt -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
28+
RUN pip3 install http://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/graphlearn-1.1.0-cp36-cp36m-linux_x86_64.whl -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
29+
RUN pip3 install http://easyrec.oss-cn-beijing.aliyuncs.com/releases/pai_automl-0.0.1rc1-py3-none-any.whl -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
30+
RUN pip3 install http://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/common_io-0.3.0-cp36-cp36m-linux_x86_64.whl -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
31+
RUN pip3 install tensorflow_probability==0.8
32+
RUN cd /EasyRec && pip install .
33+
RUN rm -rf /EasyRec
34+
RUN python -c "import easy_rec; easy_rec.help(); import pyhive; import datahub; import kafka"
35+
36+
COPY docker/hadoop_env.sh /opt/hadoop_env.sh

‎docker/Dockerfile_tf212‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
FROM tensorflow/tensorflow:2.12.0
2+
COPY docker/sources_20.04.list /etc/apt/sources.list
3+
4+
# necessary for later commands to take effect
5+
RUN md5sum /etc/apt/sources.list
6+
7+
RUN apt-get update
8+
RUN apt-get install apt-utils inetutils-ping wget curl telnet vim strace libpq-dev curl libsasl2-dev gcc g++ unzip openjdk-8-jdk -y
9+
10+
RUN mkdir /EasyRec
11+
COPY requirements /EasyRec/requirements
12+
COPY requirements.txt /EasyRec/
13+
COPY easy_rec /EasyRec/easy_rec/
14+
COPY setup.cfg /EasyRec/
15+
COPY setup.py /EasyRec/
16+
COPY MANIFEST.in /EasyRec/
17+
COPY README.md /EasyRec/
18+
COPY scripts /EasyRec/scripts
19+
20+
RUN curl "http://easyrec.oss-cn-beijing.aliyuncs.com/tools/odpscmd_public_0.45.0.zip" -o /EasyRec/odpscmd_public.zip
21+
RUN mkdir /usr/local/odps_clt/ && cd /usr/local/odps_clt/ && unzip /EasyRec/odpscmd_public.zip
22+
RUN ln -s /usr/local/odps_clt/bin/odpscmd /usr/local/bin/odpscmd
23+
24+
RUN pip3 install pystack-debugger idna kafka-python -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
25+
RUN pip3 install -r /EasyRec/requirements/runtime.txt -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
26+
RUN pip3 install -r /EasyRec/requirements/extra.txt -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
27+
RUN pip3 install https://easyrec.oss-cn-beijing.aliyuncs.com/3rdparty/graphlearn-1.1.0-cp38-cp38-linux_x86_64.whl -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
28+
# RUN pip3 install http://easyrec.oss-cn-beijing.aliyuncs.com/releases/pai_automl-0.0.1rc1-py3-none-any.whl -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
29+
RUN pip3 install tensorflow_probability==0.20.0
30+
#RUN pip3 install encodings
31+
RUN pip3 install https://dlc-task.oss-cn-hangzhou.aliyuncs.com/whl/common_io-0.4.1%2Btunnel-py2.py3-none-any.whl -i http://mirrors.aliyun.com/pypi/simple --trusted-host mirrors.aliyun.com
32+
RUN cd /EasyRec && python setup.py install
33+
RUN rm -rf /EasyRec
34+
# RUN python -c "import easy_rec; easy_rec.help(); import pyhive; import datahub; import kafka"
35+
36+
COPY docker/hadoop_env.sh /opt/hadoop_env.sh

‎examples/readme.md‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -36,14 +36,14 @@ cd EasyRec
3636

3737
-- Docker环境可选
3838
(1) `python=3.6.9` + `tenserflow=1.15.5`
39-
docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.7.4
40-
docker run -td --network host -v /local_path/EasyRec:/docker_path/EasyRec mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.7.4
39+
docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.8.5
40+
docker run -td --network host -v /local_path/EasyRec:/docker_path/EasyRec mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.8.5
4141
docker exec -it <CONTAINER_ID> bash
4242

4343

44-
(2) `python=3.8.10` + `tenserflow=2.10.0`
45-
docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.10-0.7.4
46-
docker run -td --network host -v /local_path/EasyRec:/docker_path/EasyRec mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.10-0.7.4
44+
(2) `python=3.8.10` + `tenserflow=2.12.0`
45+
docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.12-0.8.5
46+
docker run -td --network host -v /local_path/EasyRec:/docker_path/EasyRec mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.12-0.8.5
4747

4848
docker exec -it <CONTAINER_ID> bash
4949
```
@@ -56,12 +56,12 @@ cd EasyRec
5656

5757
-- Docker环境可选
5858
(1) `python=3.6.9` + `tenserflow=1.15.5`
59-
bash scripts/build_docker.sh
59+
bash scripts/build_docker_tf115.sh
6060
sudo docker run -td --network host -v /local_path:/docker_path mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-<easyrec_version>
6161

62-
(2) `python=3.8.10` + `tenserflow=2.10.0`
63-
bash scripts/build_docker_tf210.sh
64-
sudo docker run -td --network host -v /local_path:/docker_path mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.10-<easyrec_version>
62+
(2) `python=3.8.10` + `tenserflow=2.12.0`
63+
bash scripts/build_docker_tf212.sh
64+
sudo docker run -td --network host -v /local_path:/docker_path mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.12-<easyrec_version>
6565

6666
sudo docker exec -it <CONTAINER_ID> bash
6767
```

‎requirements/runtime.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
eas_prediction
1+
eas_prediction == 0.24; python_version < '3.0'
2+
eas_prediction; python_version >= '3.0'
23
future
34
matplotlib
45
numpy <= 1.23

‎scripts/build_docker_tf112.sh‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/bin/bash
2+
3+
bash scripts/gen_proto.sh
4+
if [ $? -ne 0 ]
5+
then
6+
echo "gen proto failed"
7+
exit 1
8+
fi
9+
10+
version=`grep "__version__" easy_rec/version.py | awk '{ if($1 == "__version__") print $NF}'`
11+
# strip "'"
12+
version=${version//\'/}
13+
echo "EasyRec Version: $version"
14+
15+
if [ -z "$version" ]
16+
then
17+
echo "Failed to get EasyRec version"
18+
exit 1
19+
fi
20+
21+
sudo docker build --network=host . -f docker/Dockerfile_tf112 -t mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py27-tf1.12-${version}

‎scripts/build_docker_tf115.sh‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/bin/bash
2+
3+
bash scripts/gen_proto.sh
4+
if [ $? -ne 0 ]
5+
then
6+
echo "gen proto failed"
7+
exit 1
8+
fi
9+
10+
version=`grep "__version__" easy_rec/version.py | awk '{ if($1 == "__version__") print $NF}'`
11+
# strip "'"
12+
version=${version//\'/}
13+
echo "EasyRec Version: $version"
14+
15+
if [ -z "$version" ]
16+
then
17+
echo "Failed to get EasyRec version"
18+
exit 1
19+
fi
20+
21+
sudo docker build --network=host . -f docker/Dockerfile_tf115 -t mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-${version}

‎scripts/build_docker_tf212.sh‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/bin/bash
2+
3+
bash scripts/gen_proto.sh
4+
if [ $? -ne 0 ]
5+
then
6+
echo "gen proto failed"
7+
exit 1
8+
fi
9+
10+
version=`grep "__version__" easy_rec/version.py | awk '{ if($1 == "__version__") print $NF}'`
11+
# strip "'"
12+
version=${version//\'/}
13+
echo "EasyRec Version: $version"
14+
15+
if [ -z "$version" ]
16+
then
17+
echo "Failed to get EasyRec version"
18+
exit 1
19+
fi
20+
21+
sudo docker build --network=host . -f docker/Dockerfile_tf212 -t mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.12-${version}

0 commit comments

Comments
 (0)