Skip to content

Commit d307933

Browse files
sandeepgupta12sandeepgupta12
committed
updated build script and cokerfile
1 parent 8f08ace commit d307933

File tree

3 files changed

+87
-111
lines changed

3 files changed

+87
-111
lines changed

‎.ci/docker/manywheel/Dockerfile_ppc64le‎

Lines changed: 75 additions & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -1,115 +1,102 @@
1-
# Use the manylinux_2_28 base image for ppc64le
21
FROM quay.io/pypa/manylinux_2_28_ppc64le as base
32

43
# Language variables
54
ENV LC_ALL=C.UTF-8
65
ENV LANG=C.UTF-8
76
ENV LANGUAGE=C.UTF-8
87

8+
# there is a bugfix in gcc >= 14 for precompiled headers and s390x vectorization interaction.
9+
# with earlier gcc versions test/inductor/test_cpu_cpp_wrapper.py will fail.
910
ARG DEVTOOLSET_VERSION=14
11+
# Installed needed OS packages. This is to support all
12+
# the binary builds (torch, vision, audio, text, data)
13+
RUN yum -y install epel-release
14+
RUN yum -y update
15+
RUN yum install -y \
16+
sudo \
17+
autoconf \
18+
automake \
19+
bison \
20+
bzip2 \
21+
curl \
22+
diffutils \
23+
file \
24+
git \
25+
make \
26+
patch \
27+
perl \
28+
unzip \
29+
util-linux \
30+
wget \
31+
which \
32+
xz \
33+
yasm \
34+
less \
35+
zstd \
36+
libgomp \
37+
gcc-toolset-${DEVTOOLSET_VERSION}-gcc \
38+
gcc-toolset-${DEVTOOLSET_VERSION}-gcc-c++ \
39+
gcc-toolset-${DEVTOOLSET_VERSION}-binutils \
40+
gcc-toolset-${DEVTOOLSET_VERSION}-gcc-gfortran \
41+
cmake \
42+
rust \
43+
cargo \
44+
llvm-devel \
45+
libzstd-devel \
46+
python3.12-devel \
47+
python3.12-test \
48+
python3.12-setuptools \
49+
python3.12-pip \
50+
python3-virtualenv \
51+
python3.12-pyyaml \
52+
python3.12-numpy \
53+
python3.12-wheel \
54+
python3.12-cryptography \
55+
blas-devel \
56+
openblas-devel \
57+
lapack-devel \
58+
atlas-devel \
59+
libjpeg-devel \
60+
libxslt-devel \
61+
libxml2-devel \
62+
openssl-devel \
63+
valgrind \
64+
ninja-build
1065

11-
# Create symbolic links for Python 3.12
12-
#RUN ln -sf /opt/python/cp312-cp312/bin/python3.12 /usr/bin/python3 && \
13-
# ln -sf /opt/python/cp312-cp312/bin/python3.12 /usr/bin/python
14-
15-
# Install required system dependencies
16-
RUN yum -y install epel-release && \
17-
yum -y update && \
18-
yum install -y \
19-
sudo \
20-
autoconf \
21-
automake \
22-
bison \
23-
bzip2 \
24-
curl \
25-
diffutils \
26-
file \
27-
git \
28-
make \
29-
patch \
30-
perl \
31-
unzip \
32-
util-linux \
33-
wget \
34-
which \
35-
xz \
36-
yasm \
37-
less \
38-
zstd \
39-
libgomp \
40-
gcc-toolset-${DEVTOOLSET_VERSION}-gcc \
41-
gcc-toolset-${DEVTOOLSET_VERSION}-gcc-c++ \
42-
gcc-toolset-${DEVTOOLSET_VERSION}-binutils \
43-
gcc-toolset-${DEVTOOLSET_VERSION}-gcc-gfortran \
44-
cmake \
45-
ninja-build \
46-
rust \
47-
cargo \
48-
llvm-devel \
49-
libzstd-devel \
50-
python3.12-devel \
51-
python3.12-setuptools \
52-
python3.12-pip \
53-
python3-virtualenv \
54-
python3.12-numpy \
55-
python3.12-wheel \
56-
python3.12-cryptography \
57-
blas-devel \
58-
openblas-devel \
59-
lapack-devel \
60-
atlas-devel \
61-
libjpeg-devel \
62-
libxslt-devel \
63-
libxml2-devel \
64-
openssl-devel \
65-
valgrind
66-
67-
68-
# Ensure the correct Python version is used
69-
#ENV PATH=/opt/python/cp312-cp312/bin:$PATH
70-
# Add gcc-toolset to the path
7166
ENV PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/bin:$PATH
7267
ENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib64:/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH
7368

74-
# Configure git to avoid safe directory issues
69+
# git236+ would refuse to run git commands in repos owned by other users
70+
# Which causes version check to fail, as pytorch repo is bind-mounted into the image
71+
# Override this behaviour by treating every folder as safe
72+
# For more details see https://github.com/pytorch/pytorch/issues/78659#issuecomment-1144107327
7573
RUN git config --global --add safe.directory "*"
7674

7775
# installed python doesn't have development parts. Rebuild it from scratch
7876
RUN /bin/rm -rf /opt/_internal /opt/python /usr/local/*/*
7977

8078
# EPEL for cmake
8179
FROM base as patchelf
82-
83-
# Instead of ADD ./common/install_cpython.sh ...
84-
RUN git clone --depth 1 --branch temp-gha-runner-v2 https://github.com/sandeepgupta12/pytorch.git /tmp/pytorch && \
80+
RUN git clone --depth 1 --branch temp-gha-runner-v3 https://github.com/sandeepgupta12/pytorch.git /tmp/pytorch && \
8581
cp /tmp/pytorch/.ci/docker/common/install_patchelf.sh /build_scripts/install_patchelf.sh
8682
# Install patchelf
87-
# ADD ./common/install_patchelf.sh install_patchelf.sh
83+
#ADD ./common/install_patchelf.sh install_patchelf.sh
8884
#RUN bash ./install_patchelf.sh && rm install_patchelf.sh
89-
#ENV CPYTHON_VERSIONS="3.10.1"
90-
#RUN echo "CPYTHON_VERSIONS=$CPYTHON_VERSIONS" && bash -x /build_scripts/install_cpython.sh
91-
#ENV SSL_CERT_FILE=""
92-
#RUN curl -fL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
9385
RUN bash /build_scripts/install_patchelf.sh && rm -r /build_scripts
9486
RUN cp $(which patchelf) /patchelf
9587

9688
FROM patchelf as python
97-
89+
# build python
9890
# Clone only required scripts from the PyTorch repo
9991
RUN mkdir -p /build_scripts && \
10092
cp -r /tmp/pytorch/.ci/docker/manywheel/build_scripts/* /build_scripts/ && \
10193
cp /tmp/pytorch/.ci/docker/common/install_cpython.sh /build_scripts/install_cpython.sh && \
10294
rm -rf /tmp/pytorch
103-
# build python
10495
#COPY manywheel/build_scripts /build_scripts
10596
#ADD ./common/install_cpython.sh /build_scripts/install_cpython.sh
10697
ENV SSL_CERT_FILE=
107-
# # Run the build script
108-
RUN bash /build_scripts/build.sh && rm -r /build_scripts
10998
#RUN bash build_scripts/build.sh && rm -r build_scripts
110-
#RUN bash build_scripts/build.sh || (echo "Checksum verification failed!" && exit 1)
111-
112-
99+
RUN bash /build_scripts/build.sh && rm -r /build_scripts
113100

114101
FROM base as final
115102
COPY --from=python /opt/python /opt/python
@@ -120,32 +107,20 @@ COPY --from=patchelf /usr/local/bin/patchelf /usr/local/
120107
RUN alternatives --set python /usr/bin/python3.12
121108
RUN alternatives --set python3 /usr/bin/python3.12
122109

123-
RUN /opt/python/cp312-cp312/bin/python3.12 -m ensurepip && \
124-
/opt/python/cp312-cp312/bin/python3.12 -m pip install --upgrade pip setuptools wheel
110+
RUN pip-3.12 install typing_extensions
125111

126-
# Use pip-3.12 to install clean versions of needed packages (avoid yum-installed PyYAML)
127-
RUN pip-3.12 install --no-cache-dir \
128-
typing_extensions \
129-
pyyaml==6.0.2 \
130-
requests==2.27.1
131-
132-
#RUN pip-3.12 install typing_extensions
112+
ENTRYPOINT []
113+
CMD ["/bin/bash"]
133114

134-
# Install required Python packages
135-
#RUN pip install --upgrade pip
136-
# RUN pip-3.12 install typing_extensions pyyaml setuptools
137-
138-
# Install test dependencies
115+
# install test dependencies:
116+
# - grpcio requires system openssl, bundled crypto fails to build
139117
RUN dnf install -y \
140-
protobuf-devel \
141-
protobuf-c-devel \
142-
protobuf-lite-devel \
143-
wget \
144-
patch
118+
hdf5-devel \
119+
python3-h5py \
120+
git
121+
145122

146-
# Set default entrypoint
147-
ENTRYPOINT []
148-
CMD ["/bin/bash"]
123+
#RUN env GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=True pip3 install grpcio
149124

150125
# cmake-3.28.0 from pip for onnxruntime
151-
RUN python3 -m pip install cmake==3.28.0
126+
RUN python3 -mpip install cmake==3.28.0

‎.ci/pytorch/build.sh‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
#!/bin/bash
22

33
set -ex -o pipefail
4-
which python
5-
export PATH="/opt/python/cp312-cp312/bin:$PATH"
6-
export PYTHON_EXECUTABLE="/opt/python/cp312-cp312/bin/python3.12"
7-
which python
4+
# which python
5+
# export PATH="/opt/python/cp312-cp312/bin:$PATH"
6+
# export PYTHON_EXECUTABLE="/opt/python/cp312-cp312/bin/python3.12"
7+
# which python
88
# Required environment variable: $BUILD_ENVIRONMENT
99
# (This is set by default in the Docker images we build, so you don't
1010
# need to set it yourself.
@@ -258,7 +258,8 @@ else
258258
# set only when building other architectures
259259
# or building non-XLA tests.
260260
if [[ "$BUILD_ENVIRONMENT" != *rocm* &&
261-
"$BUILD_ENVIRONMENT" != *xla* ]]; then
261+
"$BUILD_ENVIRONMENT" != *xla* &&
262+
"$BUILD_ENVIRONMENT" != *ppc64le* ]]; then
262263
# Install numpy-2.0.2 for builds which are backward compatible with 1.X
263264
python -mpip install numpy==2.0.2
264265

@@ -267,7 +268,7 @@ else
267268
if [[ "$USE_SPLIT_BUILD" == "true" ]]; then
268269
python3 tools/packaging/split_wheel.py bdist_wheel
269270
else
270-
WERROR=1 $PYTHON_EXECUTABLE setup.py bdist_wheel
271+
WERROR=1 python setup.py bdist_wheel
271272
fi
272273
else
273274
python setup.py clean
@@ -278,7 +279,7 @@ else
278279
echo "USE_SPLIT_BUILD cannot be used with xla or rocm"
279280
exit 1
280281
else
281-
$PYTHON_EXECUTABLE setup.py bdist_wheel
282+
python setup.py bdist_wheel
282283
fi
283284
fi
284285
pip_install_whl "$(echo dist/*.whl)"

‎.github/workflows/_linux-build.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -346,11 +346,11 @@ jobs:
346346
if [[ ${BUILD_ENVIRONMENT} == *"s390x"* ]]; then
347347
docker exec -t "${container_name}" sh -c "python3 -m pip install -r requirements.txt"
348348
fi
349-
# if [[ ${BUILD_ENVIRONMENT} == *"ppc64le"* ]]; then
349+
if [[ ${BUILD_ENVIRONMENT} == *"ppc64le"* ]]; then
350350
351-
# docker exec -t "${container_name}" sh -c "python3 -m pip install -r requirements.txt"
352-
# docker exec -t "${container_name}" sh -c "python3 -m pip install -r requirements-build.txt"
353-
# fi
351+
docker exec -t "${container_name}" sh -c "python3 -m pip install -r requirements.txt"
352+
docker exec -t "${container_name}" sh -c "python3 -m pip install -r requirements-build.txt"
353+
fi
354354
355355
356356
docker exec -t "${container_name}" sh -c '.ci/pytorch/build.sh'

0 commit comments

Comments
 (0)