Skip to content

Commit 6aedb8a

Browse files
dhonnappa-amdAMD AMD
authored andcommitted
Cherry-picked commit with merge conflict
1 parent 55c9130 commit 6aedb8a

File tree

3,356 files changed

+216327
-70
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

3,356 files changed

+216327
-70
lines changed

.bazelrc

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,11 @@ build --cxxopt=--std=c++17
22
build --copt=-I.
33
# Bazel does not support including its cc_library targets as system
44
# headers. We work around this for generated code
5+
<<<<<<< HEAD
56
# (e.g. torch/headeronly/macros/cmake_macros.h) by making the generated directory a
7+
=======
8+
# (e.g. c10/macros/cmake_macros.h) by making the generated directory a
9+
>>>>>>> 245bf6edbc ([AUTOGENERATED] [release/2.8] skip convolution tests on Navi4x (#2675))
610
# system include path.
711
build --copt=-isystem --copt bazel-out/k8-fastbuild/bin
812
build --copt=-isystem --copt bazel-out/darwin-fastbuild/bin

.ci/aarch64_linux/aarch64_ci_build.sh

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ set -eux -o pipefail
33

44
GPU_ARCH_VERSION=${GPU_ARCH_VERSION:-}
55

6+
<<<<<<< HEAD
67
# Set CUDA architecture lists to match x86 build_cuda.sh
78
if [[ "$GPU_ARCH_VERSION" == *"12.6"* ]]; then
89
export TORCH_CUDA_ARCH_LIST="8.0;9.0"
@@ -17,6 +18,10 @@ if [[ "$DESIRED_CUDA" == *"13"* ]]; then
1718
export TORCH_NVCC_FLAGS="-compress-mode=size"
1819
# Bundle ptxas into the cu13 wheel, see https://github.com/pytorch/pytorch/issues/163801
1920
export BUILD_BUNDLE_PTXAS=1
21+
=======
22+
if [[ "$GPU_ARCH_VERSION" == *"12.9"* ]]; then
23+
export TORCH_CUDA_ARCH_LIST="8.0;9.0;10.0;12.0"
24+
>>>>>>> 245bf6edbc ([AUTOGENERATED] [release/2.8] skip convolution tests on Navi4x (#2675))
2025
fi
2126

2227
SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
@@ -30,14 +35,19 @@ cd /
3035
# on the mounted pytorch repo
3136
git config --global --add safe.directory /pytorch
3237
pip install -r /pytorch/requirements.txt
38+
<<<<<<< HEAD
3339
pip install auditwheel==6.2.0 wheel
40+
=======
41+
pip install auditwheel==6.2.0
42+
>>>>>>> 245bf6edbc ([AUTOGENERATED] [release/2.8] skip convolution tests on Navi4x (#2675))
3443
if [ "$DESIRED_CUDA" = "cpu" ]; then
3544
echo "BASE_CUDA_VERSION is not set. Building cpu wheel."
3645
#USE_PRIORITIZED_TEXT_FOR_LD for enable linker script optimization https://github.com/pytorch/pytorch/pull/121975/files
3746
USE_PRIORITIZED_TEXT_FOR_LD=1 python /pytorch/.ci/aarch64_linux/aarch64_wheel_ci_build.py --enable-mkldnn
3847
else
3948
echo "BASE_CUDA_VERSION is set to: $DESIRED_CUDA"
4049
export USE_SYSTEM_NCCL=1
50+
<<<<<<< HEAD
4151

4252
# Check if we should use NVIDIA libs from PyPI (similar to x86 build_cuda.sh logic)
4353
if [[ -z "$PYTORCH_EXTRA_INSTALL_REQUIREMENTS" ]]; then
@@ -48,6 +58,8 @@ else
4858
export USE_NVIDIA_PYPI_LIBS=1
4959
fi
5060

61+
=======
62+
>>>>>>> 245bf6edbc ([AUTOGENERATED] [release/2.8] skip convolution tests on Navi4x (#2675))
5163
#USE_PRIORITIZED_TEXT_FOR_LD for enable linker script optimization https://github.com/pytorch/pytorch/pull/121975/files
5264
USE_PRIORITIZED_TEXT_FOR_LD=1 python /pytorch/.ci/aarch64_linux/aarch64_wheel_ci_build.py --enable-mkldnn --enable-cuda
5365
fi

.ci/aarch64_linux/aarch64_wheel_ci_build.py

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ def replace_tag(filename) -> None:
6969
f.writelines(lines)
7070

7171

72+
<<<<<<< HEAD
7273
def patch_library_rpath(
7374
folder: str,
7475
lib_name: str,
@@ -131,11 +132,14 @@ def copy_and_patch_library(
131132
patch_library_rpath(folder, lib_name, use_nvidia_pypi_libs, desired_cuda)
132133

133134

135+
=======
136+
>>>>>>> 245bf6edbc ([AUTOGENERATED] [release/2.8] skip convolution tests on Navi4x (#2675))
134137
def package_cuda_wheel(wheel_path, desired_cuda) -> None:
135138
"""
136139
Package the cuda wheel libraries
137140
"""
138141
folder = os.path.dirname(wheel_path)
142+
<<<<<<< HEAD
139143
os.mkdir(f"{folder}/tmp")
140144
os.system(f"unzip {wheel_path} -d {folder}/tmp")
141145
# Delete original wheel since it will be repackaged
@@ -249,15 +253,77 @@ def package_cuda_wheel(wheel_path, desired_cuda) -> None:
249253
# Copy libraries to unzipped_folder/torch/lib
250254
for lib_path in libs_to_copy:
251255
copy_and_patch_library(lib_path, folder, use_nvidia_pypi_libs, desired_cuda)
256+
=======
257+
wheelname = os.path.basename(wheel_path)
258+
os.mkdir(f"{folder}/tmp")
259+
os.system(f"unzip {wheel_path} -d {folder}/tmp")
260+
libs_to_copy = [
261+
"/usr/local/cuda/extras/CUPTI/lib64/libcupti.so.12",
262+
"/usr/local/cuda/lib64/libcudnn.so.9",
263+
"/usr/local/cuda/lib64/libcublas.so.12",
264+
"/usr/local/cuda/lib64/libcublasLt.so.12",
265+
"/usr/local/cuda/lib64/libcudart.so.12",
266+
"/usr/local/cuda/lib64/libcufft.so.11",
267+
"/usr/local/cuda/lib64/libcusparse.so.12",
268+
"/usr/local/cuda/lib64/libcusparseLt.so.0",
269+
"/usr/local/cuda/lib64/libcusolver.so.11",
270+
"/usr/local/cuda/lib64/libcurand.so.10",
271+
"/usr/local/cuda/lib64/libnccl.so.2",
272+
"/usr/local/cuda/lib64/libnvJitLink.so.12",
273+
"/usr/local/cuda/lib64/libnvrtc.so.12",
274+
"/usr/local/cuda/lib64/libcudnn_adv.so.9",
275+
"/usr/local/cuda/lib64/libcudnn_cnn.so.9",
276+
"/usr/local/cuda/lib64/libcudnn_graph.so.9",
277+
"/usr/local/cuda/lib64/libcudnn_ops.so.9",
278+
"/usr/local/cuda/lib64/libcudnn_engines_runtime_compiled.so.9",
279+
"/usr/local/cuda/lib64/libcudnn_engines_precompiled.so.9",
280+
"/usr/local/cuda/lib64/libcudnn_heuristic.so.9",
281+
"/lib64/libgomp.so.1",
282+
"/usr/lib64/libgfortran.so.5",
283+
"/acl/build/libarm_compute.so",
284+
"/acl/build/libarm_compute_graph.so",
285+
"/usr/local/lib/libnvpl_lapack_lp64_gomp.so.0",
286+
"/usr/local/lib/libnvpl_blas_lp64_gomp.so.0",
287+
"/usr/local/lib/libnvpl_lapack_core.so.0",
288+
"/usr/local/lib/libnvpl_blas_core.so.0",
289+
]
290+
291+
if "129" in desired_cuda:
292+
libs_to_copy += [
293+
"/usr/local/cuda/lib64/libnvrtc-builtins.so.12.9",
294+
"/usr/local/cuda/lib64/libcufile.so.0",
295+
"/usr/local/cuda/lib64/libcufile_rdma.so.1",
296+
]
297+
298+
# Copy libraries to unzipped_folder/a/lib
299+
for lib_path in libs_to_copy:
300+
lib_name = os.path.basename(lib_path)
301+
shutil.copy2(lib_path, f"{folder}/tmp/torch/lib/{lib_name}")
302+
os.system(
303+
f"cd {folder}/tmp/torch/lib/; "
304+
f"patchelf --set-rpath '$ORIGIN' --force-rpath {folder}/tmp/torch/lib/{lib_name}"
305+
)
306+
>>>>>>> 245bf6edbc ([AUTOGENERATED] [release/2.8] skip convolution tests on Navi4x (#2675))
252307

253308
# Make sure the wheel is tagged with manylinux_2_28
254309
for f in os.scandir(f"{folder}/tmp/"):
255310
if f.is_dir() and f.name.endswith(".dist-info"):
256311
replace_tag(f"{f.path}/WHEEL")
257312
break
258313

314+
<<<<<<< HEAD
259315
os.system(f"wheel pack {folder}/tmp/ -d {folder}")
260316
os.system(f"rm -rf {folder}/tmp/")
317+
=======
318+
os.mkdir(f"{folder}/cuda_wheel")
319+
os.system(f"cd {folder}/tmp/; zip -r {folder}/cuda_wheel/{wheelname} *")
320+
shutil.move(
321+
f"{folder}/cuda_wheel/{wheelname}",
322+
f"{folder}/{wheelname}",
323+
copy_function=shutil.copy2,
324+
)
325+
os.system(f"rm -rf {folder}/tmp/ {folder}/cuda_wheel/")
326+
>>>>>>> 245bf6edbc ([AUTOGENERATED] [release/2.8] skip convolution tests on Navi4x (#2675))
261327

262328

263329
def complete_wheel(folder: str) -> str:
@@ -280,7 +346,18 @@ def complete_wheel(folder: str) -> str:
280346
f"/{folder}/dist/{repaired_wheel_name}",
281347
)
282348
else:
349+
<<<<<<< HEAD
283350
repaired_wheel_name = list_dir(f"/{folder}/dist")[0]
351+
=======
352+
repaired_wheel_name = wheel_name.replace(
353+
"linux_aarch64", "manylinux_2_28_aarch64"
354+
)
355+
print(f"Renaming {wheel_name} wheel to {repaired_wheel_name}")
356+
os.rename(
357+
f"/{folder}/dist/{wheel_name}",
358+
f"/{folder}/dist/{repaired_wheel_name}",
359+
)
360+
>>>>>>> 245bf6edbc ([AUTOGENERATED] [release/2.8] skip convolution tests on Navi4x (#2675))
284361

285362
print(f"Copying {repaired_wheel_name} to artifacts")
286363
shutil.copy2(
@@ -320,6 +397,7 @@ def parse_arguments():
320397
build_vars = "CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000 "
321398
# MAX_JOB=5 is not required for CPU backend (see commit 465d98b)
322399
if enable_cuda:
400+
<<<<<<< HEAD
323401
build_vars += "MAX_JOBS=5 "
324402

325403
# Handle PyPI NVIDIA libraries vs bundled libraries
@@ -331,6 +409,9 @@ def parse_arguments():
331409
else:
332410
print("Configuring build for bundled NVIDIA libraries")
333411
# Keep existing static linking approach - already configured above
412+
=======
413+
build_vars = "MAX_JOBS=5 " + build_vars
414+
>>>>>>> 245bf6edbc ([AUTOGENERATED] [release/2.8] skip convolution tests on Navi4x (#2675))
334415

335416
override_package_version = os.getenv("OVERRIDE_PACKAGE_VERSION")
336417
desired_cuda = os.getenv("DESIRED_CUDA")

.ci/aarch64_linux/build_aarch64_wheel.py

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -438,7 +438,13 @@ def build_torchvision(
438438
)
439439
build_vars += f"BUILD_VERSION={version}.dev{build_date}"
440440
elif build_version is not None:
441+
<<<<<<< HEAD
441442
build_vars += f"BUILD_VERSION={build_version} PYTORCH_VERSION={branch[1:].split('-', maxsplit=1)[0]}"
443+
=======
444+
build_vars += (
445+
f"BUILD_VERSION={build_version} PYTORCH_VERSION={branch[1:].split('-')[0]}"
446+
)
447+
>>>>>>> 245bf6edbc ([AUTOGENERATED] [release/2.8] skip convolution tests on Navi4x (#2675))
442448
if host.using_docker():
443449
build_vars += " CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000"
444450

@@ -493,7 +499,13 @@ def build_torchdata(
493499
)
494500
build_vars += f"BUILD_VERSION={version}.dev{build_date}"
495501
elif build_version is not None:
502+
<<<<<<< HEAD
496503
build_vars += f"BUILD_VERSION={build_version} PYTORCH_VERSION={branch[1:].split('-', maxsplit=1)[0]}"
504+
=======
505+
build_vars += (
506+
f"BUILD_VERSION={build_version} PYTORCH_VERSION={branch[1:].split('-')[0]}"
507+
)
508+
>>>>>>> 245bf6edbc ([AUTOGENERATED] [release/2.8] skip convolution tests on Navi4x (#2675))
497509
if host.using_docker():
498510
build_vars += " CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000"
499511

@@ -549,7 +561,13 @@ def build_torchtext(
549561
)
550562
build_vars += f"BUILD_VERSION={version}.dev{build_date}"
551563
elif build_version is not None:
564+
<<<<<<< HEAD
552565
build_vars += f"BUILD_VERSION={build_version} PYTORCH_VERSION={branch[1:].split('-', maxsplit=1)[0]}"
566+
=======
567+
build_vars += (
568+
f"BUILD_VERSION={build_version} PYTORCH_VERSION={branch[1:].split('-')[0]}"
569+
)
570+
>>>>>>> 245bf6edbc ([AUTOGENERATED] [release/2.8] skip convolution tests on Navi4x (#2675))
553571
if host.using_docker():
554572
build_vars += " CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000"
555573

@@ -607,7 +625,13 @@ def build_torchaudio(
607625
)
608626
build_vars += f"BUILD_VERSION={version}.dev{build_date}"
609627
elif build_version is not None:
628+
<<<<<<< HEAD
610629
build_vars += f"BUILD_VERSION={build_version} PYTORCH_VERSION={branch[1:].split('-', maxsplit=1)[0]}"
630+
=======
631+
build_vars += (
632+
f"BUILD_VERSION={build_version} PYTORCH_VERSION={branch[1:].split('-')[0]}"
633+
)
634+
>>>>>>> 245bf6edbc ([AUTOGENERATED] [release/2.8] skip convolution tests on Navi4x (#2675))
611635
if host.using_docker():
612636
build_vars += " CMAKE_SHARED_LINKER_FLAGS=-Wl,-z,max-page-size=0x10000"
613637

.ci/docker/README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ See `build.sh` for valid build environments (it's the giant switch).
3636
# Set flags (see build.sh) and build image
3737
sudo bash -c 'TRITON=1 ./build.sh pytorch-linux-bionic-py3.8-gcc9 -t myimage:latest
3838
```
39+
<<<<<<< HEAD
3940
4041
## [Guidance] Adding a New Base Docker Image
4142
@@ -137,3 +138,5 @@ If your new Docker image needs a library installed from a specific pinned commit
137138

138139
The `docker-builds.yml` workflow pre-builds the Docker images whenever changes occur in the `.ci/docker/` directory. This includes the
139140
pinned commit updates.
141+
=======
142+
>>>>>>> 245bf6edbc ([AUTOGENERATED] [release/2.8] skip convolution tests on Navi4x (#2675))

.ci/docker/almalinux/Dockerfile

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,10 +64,13 @@ FROM cuda as cuda12.9
6464
RUN bash ./install_cuda.sh 12.9
6565
ENV DESIRED_CUDA=12.9
6666

67+
<<<<<<< HEAD
6768
FROM cuda as cuda13.0
6869
RUN bash ./install_cuda.sh 13.0
6970
ENV DESIRED_CUDA=13.0
7071

72+
=======
73+
>>>>>>> 245bf6edbc ([AUTOGENERATED] [release/2.8] skip convolution tests on Navi4x (#2675))
7174
FROM ${ROCM_IMAGE} as rocm
7275
ENV PYTORCH_ROCM_ARCH="gfx900;gfx906;gfx908;gfx90a;gfx942;gfx1030;gfx1100;gfx1101;gfx1102;gfx1200;gfx1201"
7376
ADD ./common/install_mkl.sh install_mkl.sh
@@ -80,10 +83,17 @@ ADD ./common/install_mnist.sh install_mnist.sh
8083
RUN bash ./install_mnist.sh
8184

8285
FROM base as all_cuda
86+
<<<<<<< HEAD
8387
COPY --from=cuda12.6 /usr/local/cuda-12.6 /usr/local/cuda-12.6
8488
COPY --from=cuda12.8 /usr/local/cuda-12.8 /usr/local/cuda-12.8
8589
COPY --from=cuda12.9 /usr/local/cuda-12.9 /usr/local/cuda-12.9
8690
COPY --from=cuda13.0 /usr/local/cuda-13.0 /usr/local/cuda-13.0
91+
=======
92+
COPY --from=cuda11.8 /usr/local/cuda-11.8 /usr/local/cuda-11.8
93+
COPY --from=cuda12.6 /usr/local/cuda-12.6 /usr/local/cuda-12.6
94+
COPY --from=cuda12.8 /usr/local/cuda-12.8 /usr/local/cuda-12.8
95+
COPY --from=cuda12.9 /usr/local/cuda-12.9 /usr/local/cuda-12.9
96+
>>>>>>> 245bf6edbc ([AUTOGENERATED] [release/2.8] skip convolution tests on Navi4x (#2675))
8797

8898
# Final step
8999
FROM ${BASE_TARGET} as final

0 commit comments

Comments
 (0)