Skip to content

Commit cd6e2b9

Browse files
rhdongrhdong
committed
[CI] clear CI cache folder and reduce size of CI docker img
1 parent 4020aa9 commit cd6e2b9

6 files changed

+321
-6
lines changed

‎.github/workflows/make_wheel_Linux_x86.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ else
1212
fi
1313

1414
if [ $TF_VERSION == "2.6.3" ] || [ $TF_VERSION == "2.8.3" ] ; then
15-
export BUILD_IMAGE="tfra/nosla-cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-multipython"
15+
export BUILD_IMAGE="tfra/nosla-cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-python$PY_VERSION"
1616
export TF_CUDA_VERSION="11.2"
1717
export TF_CUDNN_VERSION="8.1"
1818
elif [ $TF_VERSION == "2.4.1" ] ; then

‎.github/workflows/release.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,8 @@ jobs:
6464
tf-need-cuda: '1'
6565
fail-fast: false
6666
steps:
67+
- name: clear cache folder
68+
run: rm -rf /usr/share/dotnet /opt/ghc "/usr/local/share/boost"
6769
- uses: actions/[email protected]
6870
id: author-date
6971
with:

‎tools/docker/cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-multipython.Dockerfile‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -87,20 +87,15 @@ WORKDIR /tmp/
8787
COPY install/install_nccl.sh /install/
8888
RUN /install/install_nccl.sh "2.8.4-1+cuda11.2"
8989

90-
COPY install/install_rocksdb.sh /install/
91-
RUN /install/install_rocksdb.sh "6.22.1"
92-
9390
COPY install/install_bazel.sh /install/
9491
RUN /install/install_bazel.sh "5.1.1"
9592

9693
COPY install/build_and_install_python.sh /install/
9794
RUN /install/build_and_install_python.sh "3.7.7"
9895
RUN /install/build_and_install_python.sh "3.8.2"
9996
RUN /install/build_and_install_python.sh "3.9.7"
100-
RUN /install/build_and_install_python.sh "3.10.6"
10197

10298
COPY install/install_pip_packages_by_version.sh /install/
103-
RUN /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.10"
10499
RUN /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.9"
105100
RUN /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.8"
106101
RUN /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.7"

‎tools/docker/cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-python3.7.Dockerfile‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
# Dockerfile to build a manylinux 2010 compliant cross-compiler.
2+
#
3+
# Builds a devtoolset gcc/libstdc++ that targets manylinux 2010 compatible
4+
# glibc (2.12) and system libstdc++ (4.4).
5+
#
6+
# To push a new version, run:
7+
# $ docker build -f cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-python3.7.Dockerfile . \
8+
# --tag "tfra/nosla-cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-python3.7"
9+
# $ docker push tfra/nosla-cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-python3.7
10+
11+
FROM nvidia/cuda:11.2.1-cudnn8-devel-ubuntu20.04 as devtoolset
12+
13+
RUN chmod 777 /tmp/
14+
ENV DEBIAN_FRONTEND=noninteractive
15+
RUN apt-get update && apt-get install -y \
16+
cpio \
17+
file \
18+
flex \
19+
g++ \
20+
make \
21+
patch \
22+
rpm2cpio \
23+
unar \
24+
wget \
25+
xz-utils \
26+
libjpeg-dev \
27+
zlib1g-dev \
28+
libgflags-dev \
29+
libsnappy-dev \
30+
libbz2-dev \
31+
liblz4-dev \
32+
libzstd-dev \
33+
openssh-client \
34+
&& \
35+
rm -rf /var/lib/apt/lists/*
36+
37+
ADD devtoolset/fixlinks.sh fixlinks.sh
38+
ADD devtoolset/build_devtoolset.sh build_devtoolset.sh
39+
ADD devtoolset/rpm-patch.sh rpm-patch.sh
40+
41+
# Set up a sysroot for glibc 2.12 / libstdc++ 4.4 / devtoolset-7 in /dt7.
42+
RUN /build_devtoolset.sh devtoolset-7 /dt7
43+
# Set up a sysroot for glibc 2.12 / libstdc++ 4.4 / devtoolset-8 in /dt8.
44+
RUN /build_devtoolset.sh devtoolset-8 /dt8
45+
46+
# TODO(klimek): Split up into two different docker images.
47+
FROM nvidia/cuda:11.2.1-cudnn8-devel-ubuntu20.04
48+
COPY --from=devtoolset /dt7 /dt7
49+
COPY --from=devtoolset /dt8 /dt8
50+
51+
# Install TensorRT.
52+
RUN echo \
53+
deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 / \
54+
> /etc/apt/sources.list.d/nvidia-ml.list \
55+
&& \
56+
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F60F4B3D7FA2AF80 && \
57+
apt-get update && \
58+
rm -rf /var/lib/apt/lists/*
59+
60+
# Copy and run the install scripts.
61+
ARG DEBIAN_FRONTEND=noninteractive
62+
63+
COPY install/install_bootstrap_deb_packages.sh /install/
64+
RUN /install/install_bootstrap_deb_packages.sh
65+
66+
COPY install/install_deb_packages.sh /install/
67+
RUN /install/install_deb_packages.sh
68+
69+
# Install additional packages needed for this image:
70+
# - dependencies to build Python from source
71+
# - patchelf, as it is required by auditwheel
72+
RUN apt-get update && apt-get install -y \
73+
libbz2-dev \
74+
libffi-dev \
75+
libgdbm-dev \
76+
libncurses5-dev \
77+
libnss3-dev \
78+
libreadline-dev \
79+
patchelf \
80+
gcc-multilib \
81+
&& \
82+
rm -rf /var/lib/apt/lists/*
83+
84+
RUN chmod 777 /tmp/
85+
WORKDIR /tmp/
86+
87+
COPY install/install_nccl.sh /install/
88+
RUN /install/install_nccl.sh "2.8.4-1+cuda11.2"
89+
90+
COPY install/install_bazel.sh /install/
91+
RUN /install/install_bazel.sh "5.1.1"
92+
93+
COPY install/build_and_install_python.sh /install/
94+
RUN /install/build_and_install_python.sh "3.7.7"
95+
96+
COPY install/install_pip_packages_by_version.sh /install/
97+
RUN /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.7"
98+
99+
COPY install/use_devtoolset_7.sh /install/
100+
RUN /install/use_devtoolset_7.sh
101+
102+
COPY install/install_openmpi.sh /install/
103+
RUN /install/install_openmpi.sh "4.1.1"
104+
105+
# clean
106+
RUN rm -rf /tmp/*

‎tools/docker/cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-python3.8.Dockerfile‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
# Dockerfile to build a manylinux 2010 compliant cross-compiler.
2+
#
3+
# Builds a devtoolset gcc/libstdc++ that targets manylinux 2010 compatible
4+
# glibc (2.12) and system libstdc++ (4.4).
5+
#
6+
# To push a new version, run:
7+
# $ docker build -f cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-python3.8.Dockerfile . \
8+
# --tag "tfra/nosla-cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-python3.8"
9+
# $ docker push tfra/nosla-cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-python3.8
10+
11+
FROM nvidia/cuda:11.2.1-cudnn8-devel-ubuntu20.04 as devtoolset
12+
13+
RUN chmod 777 /tmp/
14+
ENV DEBIAN_FRONTEND=noninteractive
15+
RUN apt-get update && apt-get install -y \
16+
cpio \
17+
file \
18+
flex \
19+
g++ \
20+
make \
21+
patch \
22+
rpm2cpio \
23+
unar \
24+
wget \
25+
xz-utils \
26+
libjpeg-dev \
27+
zlib1g-dev \
28+
libgflags-dev \
29+
libsnappy-dev \
30+
libbz2-dev \
31+
liblz4-dev \
32+
libzstd-dev \
33+
openssh-client \
34+
&& \
35+
rm -rf /var/lib/apt/lists/*
36+
37+
ADD devtoolset/fixlinks.sh fixlinks.sh
38+
ADD devtoolset/build_devtoolset.sh build_devtoolset.sh
39+
ADD devtoolset/rpm-patch.sh rpm-patch.sh
40+
41+
# Set up a sysroot for glibc 2.12 / libstdc++ 4.4 / devtoolset-7 in /dt7.
42+
RUN /build_devtoolset.sh devtoolset-7 /dt7
43+
# Set up a sysroot for glibc 2.12 / libstdc++ 4.4 / devtoolset-8 in /dt8.
44+
RUN /build_devtoolset.sh devtoolset-8 /dt8
45+
46+
# TODO(klimek): Split up into two different docker images.
47+
FROM nvidia/cuda:11.2.1-cudnn8-devel-ubuntu20.04
48+
COPY --from=devtoolset /dt7 /dt7
49+
COPY --from=devtoolset /dt8 /dt8
50+
51+
# Install TensorRT.
52+
RUN echo \
53+
deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 / \
54+
> /etc/apt/sources.list.d/nvidia-ml.list \
55+
&& \
56+
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F60F4B3D7FA2AF80 && \
57+
apt-get update && \
58+
rm -rf /var/lib/apt/lists/*
59+
60+
# Copy and run the install scripts.
61+
ARG DEBIAN_FRONTEND=noninteractive
62+
63+
COPY install/install_bootstrap_deb_packages.sh /install/
64+
RUN /install/install_bootstrap_deb_packages.sh
65+
66+
COPY install/install_deb_packages.sh /install/
67+
RUN /install/install_deb_packages.sh
68+
69+
# Install additional packages needed for this image:
70+
# - dependencies to build Python from source
71+
# - patchelf, as it is required by auditwheel
72+
RUN apt-get update && apt-get install -y \
73+
libbz2-dev \
74+
libffi-dev \
75+
libgdbm-dev \
76+
libncurses5-dev \
77+
libnss3-dev \
78+
libreadline-dev \
79+
patchelf \
80+
gcc-multilib \
81+
&& \
82+
rm -rf /var/lib/apt/lists/*
83+
84+
RUN chmod 777 /tmp/
85+
WORKDIR /tmp/
86+
87+
COPY install/install_nccl.sh /install/
88+
RUN /install/install_nccl.sh "2.8.4-1+cuda11.2"
89+
90+
COPY install/install_bazel.sh /install/
91+
RUN /install/install_bazel.sh "5.1.1"
92+
93+
COPY install/build_and_install_python.sh /install/
94+
RUN /install/build_and_install_python.sh "3.8.2"
95+
96+
COPY install/install_pip_packages_by_version.sh /install/
97+
RUN /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.8"
98+
99+
COPY install/use_devtoolset_7.sh /install/
100+
RUN /install/use_devtoolset_7.sh
101+
102+
COPY install/install_openmpi.sh /install/
103+
RUN /install/install_openmpi.sh "4.1.1"
104+
105+
# clean
106+
RUN rm -rf /tmp/*

‎tools/docker/cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-python3.9.Dockerfile‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
# Dockerfile to build a manylinux 2010 compliant cross-compiler.
2+
#
3+
# Builds a devtoolset gcc/libstdc++ that targets manylinux 2010 compatible
4+
# glibc (2.12) and system libstdc++ (4.4).
5+
#
6+
# To push a new version, run:
7+
# $ docker build -f cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-python3.9.Dockerfile . \
8+
# --tag "tfra/nosla-cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-python3.9"
9+
# $ docker push tfra/nosla-cuda11.2.1-cudnn8-ubuntu20.04-manylinux2014-python3.9
10+
11+
FROM nvidia/cuda:11.2.1-cudnn8-devel-ubuntu20.04 as devtoolset
12+
13+
RUN chmod 777 /tmp/
14+
ENV DEBIAN_FRONTEND=noninteractive
15+
RUN apt-get update && apt-get install -y \
16+
cpio \
17+
file \
18+
flex \
19+
g++ \
20+
make \
21+
patch \
22+
rpm2cpio \
23+
unar \
24+
wget \
25+
xz-utils \
26+
libjpeg-dev \
27+
zlib1g-dev \
28+
libgflags-dev \
29+
libsnappy-dev \
30+
libbz2-dev \
31+
liblz4-dev \
32+
libzstd-dev \
33+
openssh-client \
34+
&& \
35+
rm -rf /var/lib/apt/lists/*
36+
37+
ADD devtoolset/fixlinks.sh fixlinks.sh
38+
ADD devtoolset/build_devtoolset.sh build_devtoolset.sh
39+
ADD devtoolset/rpm-patch.sh rpm-patch.sh
40+
41+
# Set up a sysroot for glibc 2.12 / libstdc++ 4.4 / devtoolset-7 in /dt7.
42+
RUN /build_devtoolset.sh devtoolset-7 /dt7
43+
# Set up a sysroot for glibc 2.12 / libstdc++ 4.4 / devtoolset-8 in /dt8.
44+
RUN /build_devtoolset.sh devtoolset-8 /dt8
45+
46+
# TODO(klimek): Split up into two different docker images.
47+
FROM nvidia/cuda:11.2.1-cudnn8-devel-ubuntu20.04
48+
COPY --from=devtoolset /dt7 /dt7
49+
COPY --from=devtoolset /dt8 /dt8
50+
51+
# Install TensorRT.
52+
RUN echo \
53+
deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 / \
54+
> /etc/apt/sources.list.d/nvidia-ml.list \
55+
&& \
56+
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F60F4B3D7FA2AF80 && \
57+
apt-get update && \
58+
rm -rf /var/lib/apt/lists/*
59+
60+
# Copy and run the install scripts.
61+
ARG DEBIAN_FRONTEND=noninteractive
62+
63+
COPY install/install_bootstrap_deb_packages.sh /install/
64+
RUN /install/install_bootstrap_deb_packages.sh
65+
66+
COPY install/install_deb_packages.sh /install/
67+
RUN /install/install_deb_packages.sh
68+
69+
# Install additional packages needed for this image:
70+
# - dependencies to build Python from source
71+
# - patchelf, as it is required by auditwheel
72+
RUN apt-get update && apt-get install -y \
73+
libbz2-dev \
74+
libffi-dev \
75+
libgdbm-dev \
76+
libncurses5-dev \
77+
libnss3-dev \
78+
libreadline-dev \
79+
patchelf \
80+
gcc-multilib \
81+
&& \
82+
rm -rf /var/lib/apt/lists/*
83+
84+
RUN chmod 777 /tmp/
85+
WORKDIR /tmp/
86+
87+
COPY install/install_nccl.sh /install/
88+
RUN /install/install_nccl.sh "2.8.4-1+cuda11.2"
89+
90+
COPY install/install_bazel.sh /install/
91+
RUN /install/install_bazel.sh "5.1.1"
92+
93+
COPY install/build_and_install_python.sh /install/
94+
RUN /install/build_and_install_python.sh "3.9.7"
95+
96+
COPY install/install_pip_packages_by_version.sh /install/
97+
RUN /install/install_pip_packages_by_version.sh "/usr/local/bin/pip3.9"
98+
99+
COPY install/use_devtoolset_7.sh /install/
100+
RUN /install/use_devtoolset_7.sh
101+
102+
COPY install/install_openmpi.sh /install/
103+
RUN /install/install_openmpi.sh "4.1.1"
104+
105+
# clean
106+
RUN rm -rf /tmp/*

0 commit comments

Comments
 (0)