Skip to content

Commit 541a370

Browse files
sandeepgupta12sandeepgupta12
committed
test changes
1 parent fd0f261 commit 541a370

File tree

13 files changed

+100
-34
lines changed

13 files changed

+100
-34
lines changed

‎.ci/docker/manywheel/Dockerfile_ppc64le‎

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,9 +75,34 @@ ENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib64:/op
7575
# Configure git to avoid safe directory issues
7676
RUN git config --global --add safe.directory "*"
7777

78-
# Install required Python packages
79-
RUN pip install --upgrade pip
80-
RUN pip install typing_extensions pyyaml setuptools
78+
# installed python doesn't have development parts. Rebuild it from scratch
79+
RUN /bin/rm -rf /opt/_internal /opt/python /usr/local/*/*
80+
81+
# EPEL for cmake
82+
FROM base as patchelf
83+
# Install patchelf
84+
ADD ./common/install_patchelf.sh install_patchelf.sh
85+
RUN bash ./install_patchelf.sh && rm install_patchelf.sh
86+
RUN cp $(which patchelf) /patchelf
87+
88+
FROM patchelf as python
89+
# build python
90+
COPY manywheel/build_scripts /build_scripts
91+
ADD ./common/install_cpython.sh /build_scripts/install_cpython.sh
92+
ENV SSL_CERT_FILE=
93+
RUN bash build_scripts/build.sh && rm -r build_scripts
94+
#RUN bash build_scripts/build.sh || (echo "Checksum verification failed!" && exit 1)
95+
96+
FROM base as final
97+
COPY --from=python /opt/python /opt/python
98+
COPY --from=python /opt/_internal /opt/_internal
99+
COPY --from=python /opt/python/cp39-cp39/bin/auditwheel /usr/local/bin/auditwheel
100+
COPY --from=patchelf /usr/local/bin/patchelf /usr/local/bin/patchelf
101+
102+
RUN alternatives --set python /usr/bin/python3.12
103+
RUN alternatives --set python3 /usr/bin/python3.12
104+
105+
RUN pip-3.12 install typing_extensions
81106

82107
# Install test dependencies
83108
RUN dnf install -y \

‎.ci/docker/manywheel/build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ case ${GPU_ARCH_TYPE} in
6666
MANY_LINUX_VERSION="s390x"
6767
;;
6868
cpu-ppc64le)
69-
TARGET=base
69+
TARGET=final
7070
DOCKER_TAG=ppc64le
7171
GPU_IMAGE=redhat/ubi9
7272
DOCKER_GPU_BUILD_ARG=""

‎.ci/docker/manywheel/build_scripts/manylinux1-check.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ def is_manylinux1_compatible():
55
# Only Linux, and only x86-64 / i686
66
from distutils.util import get_platform
77

8-
if get_platform() not in ["linux-x86_64", "linux-i686", "linux-s390x"]:
8+
if get_platform() not in ["linux-x86_64", "linux-i686", "linux-s390x", "linux-ppc64le"]:
99
return False
1010

1111
# Check for presence of _manylinux module

‎.ci/manywheel/build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ case "${GPU_ARCH_TYPE:-BLANK}" in
1515
rocm)
1616
bash "${SCRIPTPATH}/build_rocm.sh"
1717
;;
18-
cpu | cpu-cxx11-abi | cpu-s390x)
18+
cpu | cpu-cxx11-abi | cpu-s390x | cpu-ppc64le)
1919
bash "${SCRIPTPATH}/build_cpu.sh"
2020
;;
2121
xpu)

‎.ci/manywheel/build_common.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -368,7 +368,7 @@ for pkg in /$WHEELHOUSE_DIR/torch_no_python*.whl /$WHEELHOUSE_DIR/torch*linux*.w
368368
done
369369

370370
# create Manylinux 2_28 tag this needs to happen before regenerate the RECORD
371-
if [[ $PLATFORM == "manylinux_2_28_x86_64" && $GPU_ARCH_TYPE != "cpu-s390x" && $GPU_ARCH_TYPE != "xpu" ]]; then
371+
if [[ $PLATFORM == "manylinux_2_28_x86_64" && $GPU_ARCH_TYPE != "cpu-s390x" && $GPU_ARCH_TYPE != "cpu-ppc64le" && $GPU_ARCH_TYPE != "xpu" ]]; then
372372
wheel_file=$(echo $(basename $pkg) | sed -e 's/-cp.*$/.dist-info\/WHEEL/g')
373373
sed -i -e s#linux_x86_64#"${PLATFORM}"# $wheel_file;
374374
fi
@@ -413,7 +413,7 @@ for pkg in /$WHEELHOUSE_DIR/torch_no_python*.whl /$WHEELHOUSE_DIR/torch*linux*.w
413413
fi
414414

415415
# Rename wheel for Manylinux 2_28
416-
if [[ $PLATFORM == "manylinux_2_28_x86_64" && $GPU_ARCH_TYPE != "cpu-s390x" && $GPU_ARCH_TYPE != "xpu" ]]; then
416+
if [[ $PLATFORM == "manylinux_2_28_x86_64" && $GPU_ARCH_TYPE != "cpu-s390x" && $GPU_ARCH_TYPE != "cpu-ppc64le" && $GPU_ARCH_TYPE != "xpu" ]]; then
417417
pkg_name=$(echo $(basename $pkg) | sed -e s#linux_x86_64#"${PLATFORM}"#)
418418
zip -rq $pkg_name $PREIX*
419419
rm -f $pkg

‎.ci/manywheel/build_cpu.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ elif [[ "$OS_NAME" == *"AlmaLinux"* ]]; then
3636
elif [[ "$OS_NAME" == *"Ubuntu"* ]]; then
3737
if [[ "$(uname -m)" == "s390x" ]]; then
3838
LIBGOMP_PATH="/usr/lib/s390x-linux-gnu/libgomp.so.1"
39+
elif [[ "$(uname -m)" == "ppc64le" ]]; then
40+
LIBGOMP_PATH="/usr/lib64/libgomp.so.1"
3941
else
4042
LIBGOMP_PATH="/usr/lib/x86_64-linux-gnu/libgomp.so.1"
4143
fi

‎.ci/pytorch/check_binary.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,7 @@ fi
194194
if [[ "$PACKAGE_TYPE" == 'libtorch' ]]; then
195195
echo "Checking that MKL is available"
196196
build_and_run_example_cpp check-torch-mkl
197-
elif [[ "$(uname -m)" != "arm64" && "$(uname -m)" != "s390x" ]]; then
197+
elif [[ "$(uname -m)" != "arm64" && "$(uname -m)" != "s390x" && "$(uname -m)" != "ppc64le" ]]; then
198198
if [[ "$(uname)" != 'Darwin' || "$PACKAGE_TYPE" != *wheel ]]; then
199199
if [[ "$(uname -m)" == "aarch64" ]]; then
200200
echo "Checking that MKLDNN is available on aarch64"
@@ -218,7 +218,7 @@ if [[ "$PACKAGE_TYPE" == 'libtorch' ]]; then
218218
echo "Checking that XNNPACK is available"
219219
build_and_run_example_cpp check-torch-xnnpack
220220
else
221-
if [[ "$(uname)" != 'Darwin' || "$PACKAGE_TYPE" != *wheel ]] && [[ "$(uname -m)" != "s390x" ]]; then
221+
if [[ "$(uname)" != 'Darwin' || "$PACKAGE_TYPE" != *wheel ]] && [[ "$(uname -m)" != "s390x" ]] && [[ "$(uname -m)" != "ppc64le" ]]; then
222222
echo "Checking that XNNPACK is available"
223223
pushd /tmp
224224
python -c 'import torch.backends.xnnpack; exit(0 if torch.backends.xnnpack.enabled else 1)'
@@ -239,7 +239,7 @@ if [[ "$OSTYPE" == "msys" ]]; then
239239
fi
240240

241241
# Test that CUDA builds are setup correctly
242-
if [[ "$DESIRED_CUDA" != 'cpu' && "$DESIRED_CUDA" != 'xpu' && "$DESIRED_CUDA" != 'cpu-cxx11-abi' && "$DESIRED_CUDA" != *"rocm"* && "$(uname -m)" != "s390x" ]]; then
242+
if [[ "$DESIRED_CUDA" != 'cpu' && "$DESIRED_CUDA" != 'xpu' && "$DESIRED_CUDA" != 'cpu-cxx11-abi' && "$DESIRED_CUDA" != *"rocm"* && "$(uname -m)" != "s390x" && "$(uname -m)" != "ppc64le" ]]; then
243243
if [[ "$PACKAGE_TYPE" == 'libtorch' ]]; then
244244
build_and_run_example_cpp check-torch-cuda
245245
else

‎.github/actions/test-pytorch-binary/action.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ runs:
3434
)
3535
3636
echo "CONTAINER_NAME=${container_name}" >> "$GITHUB_ENV"
37-
if [[ "${GPU_ARCH_TYPE}" != "rocm" && "${BUILD_ENVIRONMENT}" != "linux-aarch64-binary-manywheel" && "${BUILD_ENVIRONMENT}" != "linux-s390x-binary-manywheel" && "${GPU_ARCH_TYPE}" != "xpu" ]]; then
37+
if [[ "${GPU_ARCH_TYPE}" != "rocm" && "${BUILD_ENVIRONMENT}" != "linux-aarch64-binary-manywheel" && "${BUILD_ENVIRONMENT}" != "linux-s390x-binary-manywheel" && "${BUILD_ENVIRONMENT}" != "linux-ppc64le-binary-manywheel" && "${GPU_ARCH_TYPE}" != "xpu" ]]; then
3838
# Propagate download.pytorch.org IP to container. This is only needed on Linux non aarch64 runner
3939
grep download.pytorch.org /etc/hosts | docker exec -i "${container_name}" bash -c "/bin/cat >> /etc/hosts"
4040
fi
@@ -45,9 +45,9 @@ runs:
4545
docker exec -t "${container_name}" bash -c "source ${BINARY_ENV_FILE} && bash -x /run.sh"
4646
4747
- name: Cleanup docker
48-
if: always() && (env.BUILD_ENVIRONMENT == 'linux-s390x-binary-manywheel' || env.GPU_ARCH_TYPE == 'xpu')
48+
if: always() && (env.BUILD_ENVIRONMENT == 'linux-s390x-binary-manywheel' || env.BUILD_ENVIRONMENT != 'linux-ppc64le-binary-manywheel' || env.GPU_ARCH_TYPE == 'xpu')
4949
shell: bash
5050
run: |
51-
# on s390x or xpu stop the container for clean worker stop
51+
# on s390x oc ppc64le or xpu stop the container for clean worker stop
5252
# shellcheck disable=SC2046
5353
docker stop "${{ env.CONTAINER_NAME }}" || true

‎.github/scripts/generate_binary_build_matrix.py‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@
3838

3939
CPU_S390X_ARCH = ["cpu-s390x"]
4040

41+
CPU_PPC64LE_ARCH = ["cpu-ppc64le"]
42+
4143
CUDA_AARCH64_ARCHES = ["12.8-aarch64"]
4244

4345

@@ -150,6 +152,8 @@ def arch_type(arch_version: str) -> str:
150152
return "cpu-s390x"
151153
elif arch_version in CUDA_AARCH64_ARCHES:
152154
return "cuda-aarch64"
155+
elif arch_version in CPU_PPC64LE_ARCH:
156+
return "cpu-ppc64le"
153157
else: # arch_version should always be "cpu" in this case
154158
return "cpu"
155159

@@ -174,6 +178,7 @@ def arch_type(arch_version: str) -> str:
174178
"cpu": f"pytorch/manylinux2_28-builder:cpu-{DEFAULT_TAG}",
175179
"cpu-aarch64": f"pytorch/manylinux2_28_aarch64-builder:cpu-aarch64-{DEFAULT_TAG}",
176180
"cpu-s390x": f"pytorch/manylinuxs390x-builder:cpu-s390x-{DEFAULT_TAG}",
181+
"cpu-ppc64le": f"pytorch/manylinuxppc64le-builder:cpu-ppc64le-{DEFAULT_TAG}",
177182
}
178183

179184
RELEASE = "release"
@@ -199,6 +204,7 @@ def translate_desired_cuda(gpu_arch_type: str, gpu_arch_version: str) -> str:
199204
"cpu": "cpu",
200205
"cpu-aarch64": "cpu",
201206
"cpu-s390x": "cpu",
207+
"cpu-ppc64le": "cpu",
202208
"cuda": f"cu{gpu_arch_version.replace('.', '')}",
203209
"cuda-aarch64": f"cu{gpu_arch_version.replace('-aarch64', '').replace('.', '')}",
204210
"rocm": f"rocm{gpu_arch_version}",
@@ -269,7 +275,7 @@ def generate_wheels_matrix(
269275
use_split_build: bool = False,
270276
) -> list[dict[str, str]]:
271277
package_type = "wheel"
272-
if os == "linux" or os == "linux-aarch64" or os == "linux-s390x":
278+
if os == "linux" or os == "linux-aarch64" or os == "linux-s390x" or os == "linux-ppc64le":
273279
# NOTE: We only build manywheel packages for x86_64 and aarch64 and s390x linux
274280
package_type = "manywheel"
275281

@@ -291,6 +297,10 @@ def generate_wheels_matrix(
291297
# Only want the one arch as the CPU type is different and
292298
# uses different build/test scripts
293299
arches = ["cpu-s390x"]
300+
elif os == "linux-ppc64le":
301+
# Only want the one arch as the CPU type is different and
302+
# uses different build/test scripts
303+
arches = ["cpu-ppc64le"]
294304

295305
ret: list[dict[str, str]] = []
296306
for python_version in python_versions:
@@ -301,12 +311,13 @@ def generate_wheels_matrix(
301311
if arch_version == "cpu"
302312
or arch_version == "cpu-aarch64"
303313
or arch_version == "cpu-s390x"
314+
or arch_version == "cpu-ppc64le"
304315
or arch_version == "xpu"
305316
else arch_version
306317
)
307318

308319
# TODO: Enable python 3.13t on cpu-s390x
309-
if gpu_arch_type == "cpu-s390x" and python_version == "3.13t":
320+
if (gpu_arch_type == "cpu-s390x" or gpu_arch_type == "cpu-ppc64le") and python_version == "3.13t":
310321
continue
311322

312323
if use_split_build and (

‎.github/scripts/generate_ci_workflows.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ class OperatingSystem:
102102
MACOS_ARM64 = "macos-arm64"
103103
LINUX_AARCH64 = "linux-aarch64"
104104
LINUX_S390X = "linux-s390x"
105+
LINUX_PPC64LE = "linux-ppc64le"
105106

106107

107108
LINUX_BINARY_BUILD_WORFKLOWS = [
@@ -367,6 +368,20 @@ class OperatingSystem:
367368
),
368369
]
369370

371+
PPC64LE_BINARY_BUILD_WORKFLOWS = [
372+
BinaryBuildWorkflow(
373+
os=OperatingSystem.LINUX_PPC64LE,
374+
package_type="manywheel",
375+
build_configs=generate_binary_build_matrix.generate_wheels_matrix(
376+
OperatingSystem.LINUX_PPC64LE
377+
),
378+
ciflow_config=CIFlowConfig(
379+
labels={LABEL_CIFLOW_BINARIES, LABEL_CIFLOW_BINARIES_WHEEL},
380+
isolated_workflow=True,
381+
),
382+
),
383+
]
384+
370385

371386
def main() -> None:
372387
jinja_env = jinja2.Environment(
@@ -389,6 +404,10 @@ def main() -> None:
389404
jinja_env.get_template("linux_binary_build_workflow.yml.j2"),
390405
S390X_BINARY_BUILD_WORKFLOWS,
391406
),
407+
(
408+
jinja_env.get_template("linux_binary_build_workflow.yml.j2"),
409+
PPC64LE_BINARY_BUILD_WORKFLOWS,
410+
),
392411
(
393412
jinja_env.get_template("linux_binary_build_workflow.yml.j2"),
394413
LINUX_BINARY_SMOKE_WORKFLOWS,

0 commit comments

Comments
 (0)