Skip to content

Commit 39cf51a

Browse files
jithunnair-amdethanwee1jagadish-amd
committed
[release/2.7] Update CentOS dockerfile for CentOS Stream 9 (#2056)
Validation: http://rocm-ci.amd.com/job/mainline-framework-pytorch-internal-ub22-py3.10-ci/212/ --------- Co-authored-by: ethanwee1 <[email protected]> Co-authored-by: Jagadish Krishnamoorthy <[email protected]>
1 parent c140442 commit 39cf51a

File tree

6 files changed

+186
-119
lines changed

6 files changed

+186
-119
lines changed

‎.ci/docker/centos-rocm/Dockerfile‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
ARG CENTOS_VERSION
22

3-
FROM centos:${CENTOS_VERSION}
3+
FROM quay.io/centos/centos:stream${CENTOS_VERSION}
44

5-
ARG CENTOS_VERSION
65

76
# Set AMD gpu targets to build for
87
ARG PYTORCH_ROCM_ARCH
@@ -14,19 +13,21 @@ ENV PYTORCH_ROCM_ARCH ${PYTORCH_ROCM_ARCH}
1413
COPY ./common/install_base.sh install_base.sh
1514
RUN bash ./install_base.sh && rm install_base.sh
1615

16+
#Install langpack
17+
RUN yum install -y glibc-langpack-en
18+
1719
# Update CentOS git version
1820
RUN yum -y remove git
1921
RUN yum -y remove git-*
20-
RUN yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo-1.9-1.x86_64.rpm && \
21-
sed -i 's/packages.endpoint/packages.endpointdev/' /etc/yum.repos.d/endpoint.repo
2222
RUN yum install -y git
2323

2424
# Install devtoolset
25-
ARG DEVTOOLSET_VERSION
26-
COPY ./common/install_devtoolset.sh install_devtoolset.sh
27-
RUN bash ./install_devtoolset.sh && rm install_devtoolset.sh
25+
RUN dnf install -y rpmdevtools
2826
ENV BASH_ENV "/etc/profile"
2927

28+
# Install ninja
29+
RUN dnf --enablerepo=crb install -y ninja-build
30+
3031
# (optional) Install non-default glibc version
3132
ARG GLIBC_VERSION
3233
COPY ./common/install_glibc.sh install_glibc.sh
@@ -47,7 +48,14 @@ COPY ./common/install_conda.sh install_conda.sh
4748
COPY ./common/common_utils.sh common_utils.sh
4849
RUN bash ./install_conda.sh && rm install_conda.sh common_utils.sh /opt/conda/requirements-ci.txt
4950

50-
# (optional) Install vision packages like OpenCV
51+
# (optional) Install protobuf for ONNX
52+
ARG PROTOBUF
53+
COPY ./common/install_protobuf.sh install_protobuf.sh
54+
RUN if [ -n "${PROTOBUF}" ]; then bash ./install_protobuf.sh; fi
55+
RUN rm install_protobuf.sh
56+
ENV INSTALLED_PROTOBUF ${PROTOBUF}
57+
58+
# (optional) Install vision packages like OpenCV and ffmpeg
5159
ARG VISION
5260
COPY ./common/install_vision.sh ./common/cache_vision_models.sh ./common/common_utils.sh ./
5361
RUN if [ -n "${VISION}" ]; then bash ./install_vision.sh; fi

‎.ci/docker/common/install_base.sh‎

Lines changed: 107 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -1,95 +1,105 @@
1-
#!/bin/bash
1+
#!/bin/bash
22

3-
set -ex
3+
set -ex
44

5-
install_ubuntu() {
6-
# NVIDIA dockers for RC releases use tag names like `11.0-cudnn9-devel-ubuntu18.04-rc`,
7-
# for this case we will set UBUNTU_VERSION to `18.04-rc` so that the Dockerfile could
8-
# find the correct image. As a result, here we have to check for
9-
# "$UBUNTU_VERSION" == "18.04"*
10-
# instead of
11-
# "$UBUNTU_VERSION" == "18.04"
12-
if [[ "$UBUNTU_VERSION" == "20.04"* ]]; then
13-
cmake3="cmake=3.16*"
14-
maybe_libiomp_dev=""
15-
elif [[ "$UBUNTU_VERSION" == "22.04"* ]]; then
16-
cmake3="cmake=3.22*"
17-
maybe_libiomp_dev=""
18-
elif [[ "$UBUNTU_VERSION" == "24.04"* ]]; then
19-
cmake3="cmake=3.28*"
20-
maybe_libiomp_dev=""
21-
else
22-
cmake3="cmake=3.5*"
23-
maybe_libiomp_dev="libiomp-dev"
24-
fi
5+
install_ubuntu() {
6+
# NVIDIA dockers for RC releases use tag names like `11.0-cudnn9-devel-ubuntu18.04-rc`,
7+
# for this case we will set UBUNTU_VERSION to `18.04-rc` so that the Dockerfile could
8+
# find the correct image. As a result, here we have to check for
9+
# "$UBUNTU_VERSION" == "18.04"*
10+
# instead of
11+
# "$UBUNTU_VERSION" == "18.04"
12+
if [[ "$UBUNTU_VERSION" == "20.04"* ]]; then
13+
cmake3="cmake=3.16*"
14+
maybe_libiomp_dev=""
15+
elif [[ "$UBUNTU_VERSION" == "22.04"* ]]; then
16+
cmake3="cmake=3.22*"
17+
maybe_libiomp_dev=""
18+
elif [[ "$UBUNTU_VERSION" == "24.04"* ]]; then
19+
cmake3="cmake=3.28*"
20+
maybe_libiomp_dev=""
21+
else
22+
cmake3="cmake=3.5*"
23+
maybe_libiomp_dev="libiomp-dev"
24+
fi
2525

26-
if [[ "$CLANG_VERSION" == 15 ]]; then
27-
maybe_libomp_dev="libomp-15-dev"
28-
elif [[ "$CLANG_VERSION" == 12 ]]; then
29-
maybe_libomp_dev="libomp-12-dev"
30-
elif [[ "$CLANG_VERSION" == 10 ]]; then
31-
maybe_libomp_dev="libomp-10-dev"
32-
else
33-
maybe_libomp_dev=""
34-
fi
26+
if [[ "$CLANG_VERSION" == 15 ]]; then
27+
maybe_libomp_dev="libomp-15-dev"
28+
elif [[ "$CLANG_VERSION" == 12 ]]; then
29+
maybe_libomp_dev="libomp-12-dev"
30+
elif [[ "$CLANG_VERSION" == 10 ]]; then
31+
maybe_libomp_dev="libomp-10-dev"
32+
else
33+
maybe_libomp_dev=""
34+
fi
3535

36-
# Install common dependencies
37-
apt-get update
38-
# TODO: Some of these may not be necessary
39-
ccache_deps="asciidoc docbook-xml docbook-xsl xsltproc"
40-
deploy_deps="libffi-dev libbz2-dev libreadline-dev libncurses5-dev libncursesw5-dev libgdbm-dev libsqlite3-dev uuid-dev tk-dev"
41-
numpy_deps="gfortran"
42-
apt-get install -y --no-install-recommends \
43-
$ccache_deps \
44-
$numpy_deps \
45-
${deploy_deps} \
46-
${cmake3} \
47-
apt-transport-https \
48-
autoconf \
49-
automake \
50-
build-essential \
51-
ca-certificates \
52-
curl \
53-
git \
54-
libatlas-base-dev \
55-
libc6-dbg \
56-
${maybe_libiomp_dev} \
57-
libyaml-dev \
58-
libz-dev \
59-
libjemalloc2 \
60-
libjpeg-dev \
61-
libasound2-dev \
62-
libsndfile-dev \
63-
${maybe_libomp_dev} \
64-
software-properties-common \
65-
wget \
66-
sudo \
67-
vim \
68-
jq \
69-
libtool \
70-
vim \
71-
unzip \
72-
gpg-agent \
73-
gdb \
74-
bc
36+
# Install common dependencies
37+
apt-get update
38+
# TODO: Some of these may not be necessary
39+
ccache_deps="asciidoc docbook-xml docbook-xsl xsltproc"
40+
deploy_deps="libffi-dev libbz2-dev libreadline-dev libncurses5-dev libncursesw5-dev libgdbm-dev libsqlite3-dev uuid-dev tk-dev"
41+
numpy_deps="gfortran"
42+
apt-get install -y --no-install-recommends \
43+
$ccache_deps \
44+
$numpy_deps \
45+
${deploy_deps} \
46+
${cmake3} \
47+
apt-transport-https \
48+
autoconf \
49+
automake \
50+
build-essential \
51+
ca-certificates \
52+
curl \
53+
git \
54+
libatlas-base-dev \
55+
libc6-dbg \
56+
${maybe_libiomp_dev} \
57+
libyaml-dev \
58+
libz-dev \
59+
libjemalloc2 \
60+
libjpeg-dev \
61+
libasound2-dev \
62+
libsndfile-dev \
63+
${maybe_libomp_dev} \
64+
software-properties-common \
65+
wget \
66+
sudo \
67+
vim \
68+
jq \
69+
libtool \
70+
vim \
71+
unzip \
72+
gpg-agent \
73+
gdb \
74+
bc
7575

76-
# Should resolve issues related to various apt package repository cert issues
77-
# see: https://github.com/pytorch/pytorch/issues/65931
78-
apt-get install -y libgnutls30
76+
# Should resolve issues related to various apt package repository cert issues
77+
# see: https://github.com/pytorch/pytorch/issues/65931
78+
apt-get install -y libgnutls30
7979

80-
# Cleanup package manager
81-
apt-get autoclean && apt-get clean
82-
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
83-
}
80+
# Cleanup package manager
81+
apt-get autoclean && apt-get clean
82+
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
83+
}
8484

85-
install_centos() {
86-
# Need EPEL for many packages we depend on.
87-
# See http://fedoraproject.org/wiki/EPEL
88-
yum --enablerepo=extras install -y epel-release
85+
install_centos() {
86+
# Need EPEL for many packages we depend on.
87+
# See http://fedoraproject.org/wiki/EPEL
88+
# extras repo is not there for CentOS 9 and epel-release is already part of repo list
89+
if [[ $OS_VERSION == 9 ]]; then
90+
yum install -y epel-release
91+
ALLOW_ERASE="--allowerasing"
92+
else
93+
yum --enablerepo=extras install -y epel-release
94+
ALLOW_ERASE=""
95+
fi
8996

90-
ccache_deps="asciidoc docbook-dtds docbook-style-xsl libxslt"
91-
numpy_deps="gcc-gfortran"
92-
yum install -y \
97+
ccache_deps="asciidoc docbook-dtds docbook-style-xsl libxslt"
98+
numpy_deps="gcc-gfortran"
99+
# Note: protobuf-c-{compiler,devel} on CentOS are too old to be used
100+
# for Caffe2. That said, we still install them to make sure the build
101+
# system opts to build/use protoc and libprotobuf from third-party.
102+
yum install -y $ALLOW_ERASE \
93103
$ccache_deps \
94104
$numpy_deps \
95105
autoconf \
@@ -106,24 +116,34 @@ install_centos() {
106116
glibc-headers \
107117
glog-devel \
108118
libstdc++-devel \
109-
libsndfile-devel \
110119
make \
111-
opencv-devel \
112120
sudo \
113121
wget \
114122
vim \
115123
unzip \
116124
gdb
117125

126+
if [[ $OS_VERSION == 9 ]]
127+
then
128+
dnf --enablerepo=crb -y install libsndfile-devel
129+
yum install -y procps
130+
else
131+
yum install -y \
132+
opencv-devel \
133+
libsndfile-devel
134+
fi
135+
118136
# Cleanup
119137
yum clean all
120138
rm -rf /var/cache/yum
121139
rm -rf /var/lib/yum/yumdb
122140
rm -rf /var/lib/yum/history
123141
}
124142

125-
# Install base packages depending on the base OS
126143
ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
144+
OS_VERSION=$(grep -oP '(?<=^VERSION_ID=).+' /etc/os-release | tr -d '"')
145+
146+
# Install base packages depending on the base OS
127147
case "$ID" in
128148
ubuntu)
129149
install_ubuntu

‎.ci/docker/common/install_cache.sh‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,12 @@ sed -e 's|PATH="\(.*\)"|PATH="/opt/cache/bin:\1"|g' -i /etc/environment
3636
export PATH="/opt/cache/bin:$PATH"
3737

3838
# Setup compiler cache
39-
install_ubuntu
39+
if [ -n "$ROCM_VERSION" ]; then
40+
curl --retry 3 http://repo.radeon.com/misc/.sccache_amd/sccache -o /opt/cache/bin/sccache
41+
else
42+
install_ubuntu
43+
fi
44+
4045
chmod a+x /opt/cache/bin/sccache
4146

4247
function write_sccache_stub() {

‎.ci/docker/common/install_conda.sh‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,13 @@ if [ -n "$ANACONDA_PYTHON_VERSION" ]; then
4040

4141
# Prevent conda from updating to 4.14.0, which causes docker build failures
4242
# See https://hud.pytorch.org/pytorch/pytorch/commit/754d7f05b6841e555cea5a4b2c505dd9e0baec1d
43-
# Uncomment the below when resolved to track the latest conda update
44-
# as_jenkins conda update -y -n base conda
43+
# Uncomment the below when resolved to track the latest conda update,
44+
# but this is required for CentOS stream 9 builds
45+
ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
46+
OS_VERSION=$(grep -oP '(?<=^VERSION_ID=).+' /etc/os-release | tr -d '"')
47+
if [[ $ID == centos && $OS_VERSION == 9 ]]; then
48+
as_jenkins conda update -y -n base conda
49+
fi
4550

4651
if [[ $(uname -m) == "aarch64" ]]; then
4752
export SYSROOT_DEP="sysroot_linux-aarch64=2.17"

0 commit comments

Comments
 (0)