Skip to content

Commit f812eaa

Browse files
sandeepgupta12sandeepgupta12
committed
using build.sh to build image
1 parent ff1a3b4 commit f812eaa

File tree

2 files changed

+53
-100
lines changed

2 files changed

+53
-100
lines changed

‎.ci/docker/manywheel/build.sh‎

Lines changed: 52 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -1,168 +1,122 @@
11
#!/usr/bin/env bash
22
# Script used only in CD pipeline
33

4-
set -eou pipefail
4+
set -exou pipefail
55

66
TOPDIR=$(git rev-parse --show-toplevel)
77

88
image="$1"
99
shift
1010

1111
if [ -z "${image}" ]; then
12-
echo "Usage: $0 IMAGE"
12+
echo "Usage: $0 IMAGE:ARCHTAG"
1313
exit 1
1414
fi
1515

16-
DOCKER_IMAGE="pytorch/${image}"
16+
# Go from imagename:tag to tag
17+
DOCKER_TAG_PREFIX=$(echo "${image}" | awk -F':' '{print $2}')
1718

18-
DOCKER_REGISTRY="${DOCKER_REGISTRY:-docker.io}"
19+
GPU_ARCH_VERSION=""
20+
if [[ "${DOCKER_TAG_PREFIX}" == cuda* ]]; then
21+
# extract cuda version from image name. e.g. manylinux2_28-builder:cuda12.8 returns 12.8
22+
GPU_ARCH_VERSION=$(echo "${DOCKER_TAG_PREFIX}" | awk -F'cuda' '{print $2}')
23+
elif [[ "${DOCKER_TAG_PREFIX}" == rocm* ]]; then
24+
# extract rocm version from image name. e.g. manylinux2_28-builder:rocm6.2.4 returns 6.2.4
25+
GPU_ARCH_VERSION=$(echo "${DOCKER_TAG_PREFIX}" | awk -F'rocm' '{print $2}')
26+
fi
1927

20-
GPU_ARCH_TYPE=${GPU_ARCH_TYPE:-cpu}
21-
GPU_ARCH_VERSION=${GPU_ARCH_VERSION:-}
2228
MANY_LINUX_VERSION=${MANY_LINUX_VERSION:-}
2329
DOCKERFILE_SUFFIX=${DOCKERFILE_SUFFIX:-}
24-
WITH_PUSH=${WITH_PUSH:-}
2530

26-
case ${GPU_ARCH_TYPE} in
27-
cpu)
28-
TARGET=cpu_final
29-
DOCKER_TAG=cpu
30-
GPU_IMAGE=centos:7
31-
DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=9"
32-
;;
33-
cpu-manylinux_2_28)
31+
case ${image} in
32+
manylinux2_28-builder:cpu)
3433
TARGET=cpu_final
35-
DOCKER_TAG=cpu
3634
GPU_IMAGE=amd64/almalinux:8
37-
DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=11"
35+
DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=13"
3836
MANY_LINUX_VERSION="2_28"
3937
;;
40-
cpu-aarch64)
38+
manylinux2_28_aarch64-builder:cpu-aarch64)
4139
TARGET=final
42-
DOCKER_TAG=cpu-aarch64
43-
GPU_IMAGE=arm64v8/centos:7
44-
DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=10"
45-
MANY_LINUX_VERSION="aarch64"
46-
;;
47-
cpu-aarch64-2_28)
48-
TARGET=final
49-
DOCKER_TAG=cpu-aarch64
5040
GPU_IMAGE=arm64v8/almalinux:8
51-
DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=11 --build-arg NINJA_VERSION=1.12.1"
41+
DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=13 --build-arg NINJA_VERSION=1.12.1"
5242
MANY_LINUX_VERSION="2_28_aarch64"
5343
;;
54-
cpu-cxx11-abi)
44+
manylinuxcxx11-abi-builder:cpu-cxx11-abi)
5545
TARGET=final
56-
DOCKER_TAG=cpu-cxx11-abi
5746
GPU_IMAGE=""
5847
DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=9"
5948
MANY_LINUX_VERSION="cxx11-abi"
6049
;;
61-
cpu-s390x)
50+
manylinuxs390x-builder:cpu-s390x)
6251
TARGET=final
63-
DOCKER_TAG=cpu-s390x
6452
GPU_IMAGE=s390x/almalinux:8
6553
DOCKER_GPU_BUILD_ARG=""
6654
MANY_LINUX_VERSION="s390x"
6755
;;
68-
cpu-ppc64le)
56+
manylinuxppc64le-builder:cpu-ppc64le)
6957
TARGET=final
70-
DOCKER_TAG=ppc64le
7158
GPU_IMAGE=redhat/ubi9
7259
DOCKER_GPU_BUILD_ARG=""
7360
MANY_LINUX_VERSION="ppc64le"
7461
;;
75-
cuda)
62+
manylinux2_28-builder:cuda11*)
7663
TARGET=cuda_final
77-
DOCKER_TAG=cuda${GPU_ARCH_VERSION}
78-
# Keep this up to date with the minimum version of CUDA we currently support
79-
GPU_IMAGE=centos:7
80-
DOCKER_GPU_BUILD_ARG="--build-arg BASE_CUDA_VERSION=${GPU_ARCH_VERSION} --build-arg DEVTOOLSET_VERSION=9"
64+
GPU_IMAGE=amd64/almalinux:8
65+
DOCKER_GPU_BUILD_ARG="--build-arg BASE_CUDA_VERSION=${GPU_ARCH_VERSION} --build-arg DEVTOOLSET_VERSION=11"
66+
MANY_LINUX_VERSION="2_28"
8167
;;
82-
cuda-manylinux_2_28)
68+
manylinux2_28-builder:cuda12*)
8369
TARGET=cuda_final
84-
DOCKER_TAG=cuda${GPU_ARCH_VERSION}
8570
GPU_IMAGE=amd64/almalinux:8
86-
DOCKER_GPU_BUILD_ARG="--build-arg BASE_CUDA_VERSION=${GPU_ARCH_VERSION} --build-arg DEVTOOLSET_VERSION=11"
71+
DOCKER_GPU_BUILD_ARG="--build-arg BASE_CUDA_VERSION=${GPU_ARCH_VERSION} --build-arg DEVTOOLSET_VERSION=13"
8772
MANY_LINUX_VERSION="2_28"
8873
;;
89-
cuda-aarch64)
74+
manylinuxaarch64-builder:cuda*)
9075
TARGET=cuda_final
91-
DOCKER_TAG=cuda${GPU_ARCH_VERSION}
92-
GPU_IMAGE=arm64v8/centos:7
93-
DOCKER_GPU_BUILD_ARG="--build-arg BASE_CUDA_VERSION=${GPU_ARCH_VERSION} --build-arg DEVTOOLSET_VERSION=11"
76+
GPU_IMAGE=amd64/almalinux:8
77+
DOCKER_GPU_BUILD_ARG="--build-arg BASE_CUDA_VERSION=${GPU_ARCH_VERSION} --build-arg DEVTOOLSET_VERSION=13"
9478
MANY_LINUX_VERSION="aarch64"
9579
DOCKERFILE_SUFFIX="_cuda_aarch64"
9680
;;
97-
rocm|rocm-manylinux_2_28)
81+
manylinux2_28-builder:rocm*)
9882
TARGET=rocm_final
99-
DOCKER_TAG=rocm${GPU_ARCH_VERSION}
100-
GPU_IMAGE=rocm/dev-centos-7:${GPU_ARCH_VERSION}-complete
101-
DEVTOOLSET_VERSION="9"
102-
if [ ${GPU_ARCH_TYPE} == "rocm-manylinux_2_28" ]; then
103-
MANY_LINUX_VERSION="2_28"
104-
DEVTOOLSET_VERSION="11"
105-
GPU_IMAGE=rocm/dev-almalinux-8:${GPU_ARCH_VERSION}-complete
106-
fi
83+
MANY_LINUX_VERSION="2_28"
84+
DEVTOOLSET_VERSION="11"
85+
GPU_IMAGE=rocm/dev-almalinux-8:${GPU_ARCH_VERSION}-complete
10786
PYTORCH_ROCM_ARCH="gfx900;gfx906;gfx908;gfx90a;gfx942;gfx1030;gfx1100;gfx1101;gfx1102;gfx1200;gfx1201"
10887
DOCKER_GPU_BUILD_ARG="--build-arg ROCM_VERSION=${GPU_ARCH_VERSION} --build-arg PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH} --build-arg DEVTOOLSET_VERSION=${DEVTOOLSET_VERSION}"
10988
;;
110-
xpu)
89+
manylinux2_28-builder:xpu)
11190
TARGET=xpu_final
112-
DOCKER_TAG=xpu
11391
GPU_IMAGE=amd64/almalinux:8
11492
DOCKER_GPU_BUILD_ARG=" --build-arg DEVTOOLSET_VERSION=11"
11593
MANY_LINUX_VERSION="2_28"
11694
;;
11795
*)
118-
echo "ERROR: Unrecognized GPU_ARCH_TYPE: ${GPU_ARCH_TYPE}"
96+
echo "ERROR: Unrecognized image name: ${image}"
11997
exit 1
12098
;;
12199
esac
122100

123-
IMAGES=''
124-
125101
if [[ -n ${MANY_LINUX_VERSION} && -z ${DOCKERFILE_SUFFIX} ]]; then
126102
DOCKERFILE_SUFFIX=_${MANY_LINUX_VERSION}
127103
fi
128-
(
129-
set -x
130-
131-
132-
# Only activate this if in CI
133-
if [ "$(uname -m)" != "s390x" && "$(uname -m)" != "ppc64le" ] && [ -v CI ]; then
134-
# TODO: Remove LimitNOFILE=1048576 patch once https://github.com/pytorch/test-infra/issues/5712
135-
# is resolved. This patch is required in order to fix timing out of Docker build on Amazon Linux 2023.
136-
sudo sed -i s/LimitNOFILE=infinity/LimitNOFILE=1048576/ /usr/lib/systemd/system/docker.service
137-
sudo systemctl daemon-reload
138-
sudo systemctl restart docker
139-
fi
140-
141-
DOCKER_BUILDKIT=1 docker build \
142-
${DOCKER_GPU_BUILD_ARG} \
143-
--build-arg "GPU_IMAGE=${GPU_IMAGE}" \
144-
--target "${TARGET}" \
145-
-t "${DOCKER_IMAGE}" \
146-
$@ \
147-
-f "${TOPDIR}/.ci/docker/manywheel/Dockerfile${DOCKERFILE_SUFFIX}" \
148-
"${TOPDIR}/.ci/docker/"
149-
)
104+
# Only activate this if in CI
105+
if [ "$(uname -m)" != "s390x" ] && "$(uname -m)" != "ppc64le" ] && [ -v CI ]; then
106+
# TODO: Remove LimitNOFILE=1048576 patch once https://github.com/pytorch/test-infra/issues/5712
107+
# is resolved. This patch is required in order to fix timing out of Docker build on Amazon Linux 2023.
108+
sudo sed -i s/LimitNOFILE=infinity/LimitNOFILE=1048576/ /usr/lib/systemd/system/docker.service
109+
sudo systemctl daemon-reload
110+
sudo systemctl restart docker
111+
fi
150112

151-
GITHUB_REF=${GITHUB_REF:-"dev")}
152-
GIT_BRANCH_NAME=${GITHUB_REF##*/}
153-
GIT_COMMIT_SHA=${GITHUB_SHA:-$(git rev-parse HEAD)}
154-
DOCKER_IMAGE_BRANCH_TAG=${DOCKER_IMAGE}-${GIT_BRANCH_NAME}
155-
DOCKER_IMAGE_SHA_TAG=${DOCKER_IMAGE}-${GIT_COMMIT_SHA}
113+
tmp_tag=$(basename "$(mktemp -u)" | tr '[:upper:]' '[:lower:]')
156114

157-
if [[ "${WITH_PUSH}" == true ]]; then
158-
(
159-
set -x
160-
docker push "${DOCKER_IMAGE}"
161-
if [[ -n ${GITHUB_REF} ]]; then
162-
docker tag ${DOCKER_IMAGE} ${DOCKER_IMAGE_BRANCH_TAG}
163-
docker tag ${DOCKER_IMAGE} ${DOCKER_IMAGE_SHA_TAG}
164-
docker push "${DOCKER_IMAGE_BRANCH_TAG}"
165-
docker push "${DOCKER_IMAGE_SHA_TAG}"
166-
fi
167-
)
168-
fi
115+
DOCKER_BUILDKIT=1 docker build \
116+
${DOCKER_GPU_BUILD_ARG} \
117+
--build-arg "GPU_IMAGE=${GPU_IMAGE}" \
118+
--target "${TARGET}" \
119+
-t "${tmp_tag}" \
120+
$@ \
121+
-f "${TOPDIR}/.ci/docker/manywheel/Dockerfile${DOCKERFILE_SUFFIX}" \
122+
"${TOPDIR}/.ci/docker/"

‎.github/workflows/ppc64le.yml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,7 @@ jobs:
2424
- name: Build Docker image for ppc64le
2525
run: |
2626
# docker build -f .ci/docker/manywheel/Dockerfile_ppc64le -t manylinuxppc64le-builder .
27-
GPU_ARCH_TYPE=cpu-ppc64le "$(pwd)/.ci/docker/manywheel/build.sh" manylinuxppc64le-builder
28-
27+
.ci/docker/manywheel/build.sh manylinuxppc64le-builder:cpu-ppc64le -t manylinuxppc64le-builder
2928
3029
- name: Save Docker image to tarball
3130
run: |

0 commit comments

Comments
 (0)