Skip to content

Commit ee25cbe

Browse files
sandeepgupta12sandeepgupta12
authored
Merge pull request #3 from sandeepgupta12/temp-ppc64le-wheel-branch-v6
Temp ppc64le wheel branch v6
2 parents 30747b5 + ab68338 commit ee25cbe

19 files changed

+255
-42
lines changed

‎.ci/docker/manywheel/Dockerfile_ppc64le‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,9 +75,38 @@ ENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib64:/op
7575
# Configure git to avoid safe directory issues
7676
RUN git config --global --add safe.directory "*"
7777

78+
# installed python doesn't have development parts. Rebuild it from scratch
79+
RUN /bin/rm -rf /opt/_internal /opt/python /usr/local/*/*
80+
81+
# EPEL for cmake
82+
FROM base as patchelf
83+
# Install patchelf
84+
ADD ./common/install_patchelf.sh install_patchelf.sh
85+
RUN bash ./install_patchelf.sh && rm install_patchelf.sh
86+
RUN cp $(which patchelf) /patchelf
87+
88+
FROM patchelf as python
89+
# build python
90+
COPY manywheel/build_scripts /build_scripts
91+
ADD ./common/install_cpython.sh /build_scripts/install_cpython.sh
92+
ENV SSL_CERT_FILE=
93+
RUN bash build_scripts/build.sh && rm -r build_scripts
94+
#RUN bash build_scripts/build.sh || (echo "Checksum verification failed!" && exit 1)
95+
96+
FROM base as final
97+
COPY --from=python /opt/python /opt/python
98+
COPY --from=python /opt/_internal /opt/_internal
99+
COPY --from=python /opt/python/cp39-cp39/bin/auditwheel /usr/local/bin/auditwheel
100+
COPY --from=patchelf /usr/local/bin/patchelf /usr/local/bin/patchelf
101+
102+
RUN alternatives --set python /usr/bin/python3.12
103+
RUN alternatives --set python3 /usr/bin/python3.12
104+
105+
RUN pip-3.12 install typing_extensions
106+
78107
# Install required Python packages
79-
RUN pip install --upgrade pip
80-
RUN pip install typing_extensions pyyaml setuptools
108+
#RUN pip install --upgrade pip
109+
#RUN pip install typing_extensions pyyaml setuptools
81110

82111
# Install test dependencies
83112
RUN dnf install -y \

‎.ci/docker/manywheel/build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ case ${GPU_ARCH_TYPE} in
6666
MANY_LINUX_VERSION="s390x"
6767
;;
6868
cpu-ppc64le)
69-
TARGET=base
69+
TARGET=final
7070
DOCKER_TAG=ppc64le
7171
GPU_IMAGE=redhat/ubi9
7272
DOCKER_GPU_BUILD_ARG=""

‎.ci/docker/manywheel/build_scripts/build.sh‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ DEVTOOLS_HASH=a8ebeb4bed624700f727179e6ef771dafe47651131a00a78b342251415646acc
1313
PATCHELF_HASH=d9afdff4baeacfbc64861454f368b7f2c15c44d245293f7587bbf726bfe722fb
1414
CURL_ROOT=curl-7.73.0
1515
CURL_HASH=cf34fe0b07b800f1c01a499a6e8b2af548f6d0e044dca4a29d88a4bee146d131
16+
#CURL_HASH=ed444155f1fd7d72c44424f9333893e64cbd6c1cbe4489619bce99c4dda58c14
1617
AUTOCONF_ROOT=autoconf-2.69
1718
AUTOCONF_HASH=954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969
1819

@@ -38,8 +39,32 @@ yum -y install bzip2 make git patch unzip bison yasm diffutils \
3839
automake which file \
3940
${PYTHON_COMPILE_DEPS}
4041

41-
# Install newest autoconf
42-
build_autoconf $AUTOCONF_ROOT $AUTOCONF_HASH
42+
# Download and extract Autoconf
43+
curl -sLO http://ftp.gnu.org/gnu/autoconf/$AUTOCONF_ROOT.tar.gz
44+
echo "$AUTOCONF_HASH $AUTOCONF_ROOT.tar.gz" | sha256sum -c -
45+
tar -xzf $AUTOCONF_ROOT.tar.gz
46+
cd $AUTOCONF_ROOT
47+
48+
# Update config.guess and config.sub
49+
curl -o build-aux/config.guess http://git.savannah.gnu.org/cgit/config.git/plain/config.guess
50+
curl -o build-aux/config.sub http://git.savannah.gnu.org/cgit/config.git/plain/config.sub
51+
chmod +x build-aux/config.guess build-aux/config.sub
52+
53+
# Configure with the correct host
54+
./configure --host=powerpc64le-pc-linux-gnu
55+
56+
# Build and install
57+
make -j$(nproc)
58+
make install
59+
60+
# Clean up
61+
cd ..
62+
rm -rf $AUTOCONF_ROOT $AUTOCONF_ROOT.tar.gz
63+
64+
65+
# Install newest autoconf (previous method)
66+
#build_autoconf $AUTOCONF_ROOT $AUTOCONF_HASH
67+
4368
autoconf --version
4469

4570
# Compile the latest Python releases.

‎.ci/docker/manywheel/build_scripts/build_utils.sh‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,12 @@ function check_sha256sum {
2828
check_var ${fname}
2929
local sha256=$2
3030
check_var ${sha256}
31-
31+
# Compute and print actual checksum
32+
echo "✅ Expected SHA256: ${sha256}"
33+
echo "🔍 Calculating actual SHA256..."
34+
actual_sha256=$(sha256sum ${fname} | awk '{print $1}')
35+
echo "🔴 Actual SHA256: $actual_sha256"
36+
3237
echo "${sha256} ${fname}" > ${fname}.sha256
3338
sha256sum -c ${fname}.sha256
3439
rm -f ${fname}.sha256
@@ -41,7 +46,8 @@ function build_openssl {
4146
local openssl_sha256=$2
4247
check_var ${openssl_sha256}
4348
check_var ${OPENSSL_DOWNLOAD_URL}
44-
curl -sLO ${OPENSSL_DOWNLOAD_URL}/${openssl_fname}.tar.gz
49+
curl -sLO ${OPENSSL_DOWNLOAD_URL}/${openssl_fname}.tar.gz
50+
4551
check_sha256sum ${openssl_fname}.tar.gz ${openssl_sha256}
4652
tar -xzf ${openssl_fname}.tar.gz
4753
(cd ${openssl_fname} && do_openssl_build)

‎.ci/docker/manywheel/build_scripts/manylinux1-check.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ def is_manylinux1_compatible():
55
# Only Linux, and only x86-64 / i686
66
from distutils.util import get_platform
77

8-
if get_platform() not in ["linux-x86_64", "linux-i686", "linux-s390x"]:
8+
if get_platform() not in ["linux-x86_64", "linux-i686", "linux-s390x", "linux-pc64le"]:
99
return False
1010

1111
# Check for presence of _manylinux module

‎.ci/manywheel/build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ case "${GPU_ARCH_TYPE:-BLANK}" in
1515
rocm)
1616
bash "${SCRIPTPATH}/build_rocm.sh"
1717
;;
18-
cpu | cpu-cxx11-abi | cpu-s390x)
18+
cpu | cpu-cxx11-abi | cpu-s390x | cpu-ppc64le)
1919
bash "${SCRIPTPATH}/build_cpu.sh"
2020
;;
2121
xpu)

‎.ci/manywheel/build_common.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -380,7 +380,7 @@ for pkg in /$WHEELHOUSE_DIR/torch_no_python*.whl /$WHEELHOUSE_DIR/torch*linux*.w
380380
done
381381

382382
# create Manylinux 2_28 tag this needs to happen before regenerate the RECORD
383-
if [[ $PLATFORM == "manylinux_2_28_x86_64" && $GPU_ARCH_TYPE != "cpu-s390x" && $GPU_ARCH_TYPE != "xpu" ]]; then
383+
if [[ $PLATFORM == "manylinux_2_28_x86_64" && $GPU_ARCH_TYPE != "cpu-s390x" && $GPU_ARCH_TYPE != "cpu-ppc64le" && $GPU_ARCH_TYPE != "xpu" ]]; then
384384
wheel_file=$(echo $(basename $pkg) | sed -e 's/-cp.*$/.dist-info\/WHEEL/g')
385385
sed -i -e s#linux_x86_64#"${PLATFORM}"# $wheel_file;
386386
fi
@@ -425,7 +425,7 @@ for pkg in /$WHEELHOUSE_DIR/torch_no_python*.whl /$WHEELHOUSE_DIR/torch*linux*.w
425425
fi
426426

427427
# Rename wheel for Manylinux 2_28
428-
if [[ $PLATFORM == "manylinux_2_28_x86_64" && $GPU_ARCH_TYPE != "cpu-s390x" && $GPU_ARCH_TYPE != "xpu" ]]; then
428+
if [[ $PLATFORM == "manylinux_2_28_x86_64" && $GPU_ARCH_TYPE != "cpu-s390x" && $GPU_ARCH_TYPE != "cpu-ppc64le" && $GPU_ARCH_TYPE != "xpu" ]]; then
429429
pkg_name=$(echo $(basename $pkg) | sed -e s#linux_x86_64#"${PLATFORM}"#)
430430
zip -rq $pkg_name $PREIX*
431431
rm -f $pkg

‎.ci/manywheel/build_cpu.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ elif [[ "$OS_NAME" == *"AlmaLinux"* ]]; then
3636
elif [[ "$OS_NAME" == *"Ubuntu"* ]]; then
3737
if [[ "$(uname -m)" == "s390x" ]]; then
3838
LIBGOMP_PATH="/usr/lib/s390x-linux-gnu/libgomp.so.1"
39+
elif [[ "$(uname -m)" == "ppc64le" ]]; then
40+
LIBGOMP_PATH="/usr/lib64/libgomp.so.1"
3941
else
4042
LIBGOMP_PATH="/usr/lib/x86_64-linux-gnu/libgomp.so.1"
4143
fi

‎.ci/pytorch/check_binary.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -263,7 +263,7 @@ fi
263263
if [[ "$PACKAGE_TYPE" == 'libtorch' ]]; then
264264
echo "Checking that MKL is available"
265265
build_and_run_example_cpp check-torch-mkl
266-
elif [[ "$(uname -m)" != "arm64" && "$(uname -m)" != "s390x" ]]; then
266+
elif [[ "$(uname -m)" != "arm64" && "$(uname -m)" != "s390x" && "$(uname -m)" != "ppc64le" ]]; then
267267
if [[ "$(uname)" != 'Darwin' || "$PACKAGE_TYPE" != *wheel ]]; then
268268
if [[ "$(uname -m)" == "aarch64" ]]; then
269269
echo "Checking that MKLDNN is available on aarch64"
@@ -287,7 +287,7 @@ if [[ "$PACKAGE_TYPE" == 'libtorch' ]]; then
287287
echo "Checking that XNNPACK is available"
288288
build_and_run_example_cpp check-torch-xnnpack
289289
else
290-
if [[ "$(uname)" != 'Darwin' || "$PACKAGE_TYPE" != *wheel ]] && [[ "$(uname -m)" != "s390x" ]]; then
290+
if [[ "$(uname)" != 'Darwin' || "$PACKAGE_TYPE" != *wheel ]] && [[ "$(uname -m)" != "s390x" ]] && [[ "$(uname -m)" != "ppc64le" ]]; then
291291
echo "Checking that XNNPACK is available"
292292
pushd /tmp
293293
python -c 'import torch.backends.xnnpack; exit(0 if torch.backends.xnnpack.enabled else 1)'
@@ -308,7 +308,7 @@ if [[ "$OSTYPE" == "msys" ]]; then
308308
fi
309309

310310
# Test that CUDA builds are setup correctly
311-
if [[ "$DESIRED_CUDA" != 'cpu' && "$DESIRED_CUDA" != 'xpu' && "$DESIRED_CUDA" != 'cpu-cxx11-abi' && "$DESIRED_CUDA" != *"rocm"* && "$(uname -m)" != "s390x" ]]; then
311+
if [[ "$DESIRED_CUDA" != 'cpu' && "$DESIRED_CUDA" != 'xpu' && "$DESIRED_CUDA" != 'cpu-cxx11-abi' && "$DESIRED_CUDA" != *"rocm"* && "$(uname -m)" != "s390x" && "$(uname -m)" != "ppc64le" ]]; then
312312
if [[ "$PACKAGE_TYPE" == 'libtorch' ]]; then
313313
build_and_run_example_cpp check-torch-cuda
314314
else

‎.github/actions/test-pytorch-binary/action.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ runs:
3535
)
3636
3737
echo "CONTAINER_NAME=${container_name}" >> "$GITHUB_ENV"
38-
if [[ "${GPU_ARCH_TYPE}" != "rocm" && "${BUILD_ENVIRONMENT}" != "linux-aarch64-binary-manywheel" && "${BUILD_ENVIRONMENT}" != "linux-s390x-binary-manywheel" && "${GPU_ARCH_TYPE}" != "xpu" ]]; then
38+
if [[ "${GPU_ARCH_TYPE}" != "rocm" && "${BUILD_ENVIRONMENT}" != "linux-aarch64-binary-manywheel" && "${BUILD_ENVIRONMENT}" != "linux-s390x-binary-manywheel" && "${BUILD_ENVIRONMENT}" != "linux-ppc64le-binary-manywheel" && "${GPU_ARCH_TYPE}" != "xpu" ]]; then
3939
# Propagate download.pytorch.org IP to container. This is only needed on Linux non aarch64 runner
4040
grep download.pytorch.org /etc/hosts | docker exec -i "${container_name}" bash -c "/bin/cat >> /etc/hosts"
4141
fi
@@ -46,7 +46,7 @@ runs:
4646
docker exec -t "${container_name}" bash -c "source ${BINARY_ENV_FILE} && bash -x /run.sh"
4747
4848
- name: Cleanup docker
49-
if: always() && (env.BUILD_ENVIRONMENT == 'linux-s390x-binary-manywheel' || env.GPU_ARCH_TYPE == 'xpu')
49+
if: always() && (env.BUILD_ENVIRONMENT == 'linux-s390x-binary-manywheel' || env.BUILD_ENVIRONMENT != 'linux-ppc64le-binary-manywheel' || env.GPU_ARCH_TYPE == 'xpu')
5050
shell: bash
5151
run: |
5252
# on s390x or xpu stop the container for clean worker stop

0 commit comments

Comments
 (0)