Skip to content

Commit 4b61297

Browse files
sandeepgupta12sandeepgupta12
committed
updated dockerfile
1 parent 5f4a8c3 commit 4b61297

File tree

1 file changed

+96
-96
lines changed

1 file changed

+96
-96
lines changed

‎.ci/pytorch/build.sh‎

Lines changed: 96 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -301,104 +301,104 @@ else
301301
python setup.py bdist_wheel
302302
fi
303303
fi
304-
pip_install_whl "$(echo dist/*.whl)"
305-
306-
# TODO: I'm not sure why, but somehow we lose verbose commands
307-
set -x
308-
309-
assert_git_not_dirty
310-
# Copy ninja build logs to dist folder
311-
mkdir -p dist
312-
if [ -f build/.ninja_log ]; then
313-
cp build/.ninja_log dist
314-
fi
315-
316-
if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
317-
# remove sccache wrappers post-build; runtime compilation of MIOpen kernels does not yet fully support them
318-
sudo rm -f /opt/cache/bin/cc
319-
sudo rm -f /opt/cache/bin/c++
320-
sudo rm -f /opt/cache/bin/gcc
321-
sudo rm -f /opt/cache/bin/g++
322-
pushd /opt/rocm/llvm/bin
323-
if [[ -d original ]]; then
324-
sudo mv original/clang .
325-
sudo mv original/clang++ .
326-
fi
327-
sudo rm -rf original
328-
popd
329-
fi
330-
331-
CUSTOM_TEST_ARTIFACT_BUILD_DIR=${CUSTOM_TEST_ARTIFACT_BUILD_DIR:-"build/custom_test_artifacts"}
332-
CUSTOM_TEST_USE_ROCM=$([[ "$BUILD_ENVIRONMENT" == *rocm* ]] && echo "ON" || echo "OFF")
333-
CUSTOM_TEST_MODULE_PATH="${PWD}/cmake/public"
334-
mkdir -pv "${CUSTOM_TEST_ARTIFACT_BUILD_DIR}"
335-
336-
# Build custom operator tests.
337-
CUSTOM_OP_BUILD="${CUSTOM_TEST_ARTIFACT_BUILD_DIR}/custom-op-build"
338-
CUSTOM_OP_TEST="$PWD/test/custom_operator"
339-
python --version
340-
SITE_PACKAGES="$(python -c 'import site; print(";".join([x for x in site.getsitepackages()] + [x + "/torch" for x in site.getsitepackages()]))')"
341-
342-
mkdir -p "$CUSTOM_OP_BUILD"
343-
pushd "$CUSTOM_OP_BUILD"
344-
cmake "$CUSTOM_OP_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES" -DPython_EXECUTABLE="$(which python)" \
345-
-DCMAKE_MODULE_PATH="$CUSTOM_TEST_MODULE_PATH" -DUSE_ROCM="$CUSTOM_TEST_USE_ROCM"
346-
make VERBOSE=1
347-
popd
348-
assert_git_not_dirty
349-
350-
# Build jit hook tests
351-
JIT_HOOK_BUILD="${CUSTOM_TEST_ARTIFACT_BUILD_DIR}/jit-hook-build"
352-
JIT_HOOK_TEST="$PWD/test/jit_hooks"
353-
python --version
354-
SITE_PACKAGES="$(python -c 'import site; print(";".join([x for x in site.getsitepackages()] + [x + "/torch" for x in site.getsitepackages()]))')"
355-
mkdir -p "$JIT_HOOK_BUILD"
356-
pushd "$JIT_HOOK_BUILD"
357-
cmake "$JIT_HOOK_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES" -DPython_EXECUTABLE="$(which python)" \
358-
-DCMAKE_MODULE_PATH="$CUSTOM_TEST_MODULE_PATH" -DUSE_ROCM="$CUSTOM_TEST_USE_ROCM"
359-
make VERBOSE=1
360-
popd
361-
assert_git_not_dirty
362-
363-
# Build custom backend tests.
364-
CUSTOM_BACKEND_BUILD="${CUSTOM_TEST_ARTIFACT_BUILD_DIR}/custom-backend-build"
365-
CUSTOM_BACKEND_TEST="$PWD/test/custom_backend"
366-
python --version
367-
mkdir -p "$CUSTOM_BACKEND_BUILD"
368-
pushd "$CUSTOM_BACKEND_BUILD"
369-
cmake "$CUSTOM_BACKEND_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES" -DPython_EXECUTABLE="$(which python)" \
370-
-DCMAKE_MODULE_PATH="$CUSTOM_TEST_MODULE_PATH" -DUSE_ROCM="$CUSTOM_TEST_USE_ROCM"
371-
make VERBOSE=1
372-
popd
373-
assert_git_not_dirty
304+
# pip_install_whl "$(echo dist/*.whl)"
305+
306+
# # TODO: I'm not sure why, but somehow we lose verbose commands
307+
# set -x
308+
309+
# assert_git_not_dirty
310+
# # Copy ninja build logs to dist folder
311+
# mkdir -p dist
312+
# if [ -f build/.ninja_log ]; then
313+
# cp build/.ninja_log dist
314+
# fi
315+
316+
# if [[ "$BUILD_ENVIRONMENT" == *rocm* ]]; then
317+
# # remove sccache wrappers post-build; runtime compilation of MIOpen kernels does not yet fully support them
318+
# sudo rm -f /opt/cache/bin/cc
319+
# sudo rm -f /opt/cache/bin/c++
320+
# sudo rm -f /opt/cache/bin/gcc
321+
# sudo rm -f /opt/cache/bin/g++
322+
# pushd /opt/rocm/llvm/bin
323+
# if [[ -d original ]]; then
324+
# sudo mv original/clang .
325+
# sudo mv original/clang++ .
326+
# fi
327+
# sudo rm -rf original
328+
# popd
329+
# fi
330+
331+
# CUSTOM_TEST_ARTIFACT_BUILD_DIR=${CUSTOM_TEST_ARTIFACT_BUILD_DIR:-"build/custom_test_artifacts"}
332+
# CUSTOM_TEST_USE_ROCM=$([[ "$BUILD_ENVIRONMENT" == *rocm* ]] && echo "ON" || echo "OFF")
333+
# CUSTOM_TEST_MODULE_PATH="${PWD}/cmake/public"
334+
# mkdir -pv "${CUSTOM_TEST_ARTIFACT_BUILD_DIR}"
335+
336+
# # Build custom operator tests.
337+
# CUSTOM_OP_BUILD="${CUSTOM_TEST_ARTIFACT_BUILD_DIR}/custom-op-build"
338+
# CUSTOM_OP_TEST="$PWD/test/custom_operator"
339+
# python --version
340+
# SITE_PACKAGES="$(python -c 'import site; print(";".join([x for x in site.getsitepackages()] + [x + "/torch" for x in site.getsitepackages()]))')"
341+
342+
# mkdir -p "$CUSTOM_OP_BUILD"
343+
# pushd "$CUSTOM_OP_BUILD"
344+
# cmake "$CUSTOM_OP_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES" -DPython_EXECUTABLE="$(which python)" \
345+
# -DCMAKE_MODULE_PATH="$CUSTOM_TEST_MODULE_PATH" -DUSE_ROCM="$CUSTOM_TEST_USE_ROCM"
346+
# make VERBOSE=1
347+
# popd
348+
# assert_git_not_dirty
349+
350+
# # Build jit hook tests
351+
# JIT_HOOK_BUILD="${CUSTOM_TEST_ARTIFACT_BUILD_DIR}/jit-hook-build"
352+
# JIT_HOOK_TEST="$PWD/test/jit_hooks"
353+
# python --version
354+
# SITE_PACKAGES="$(python -c 'import site; print(";".join([x for x in site.getsitepackages()] + [x + "/torch" for x in site.getsitepackages()]))')"
355+
# mkdir -p "$JIT_HOOK_BUILD"
356+
# pushd "$JIT_HOOK_BUILD"
357+
# cmake "$JIT_HOOK_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES" -DPython_EXECUTABLE="$(which python)" \
358+
# -DCMAKE_MODULE_PATH="$CUSTOM_TEST_MODULE_PATH" -DUSE_ROCM="$CUSTOM_TEST_USE_ROCM"
359+
# make VERBOSE=1
360+
# popd
361+
# assert_git_not_dirty
362+
363+
# # Build custom backend tests.
364+
# CUSTOM_BACKEND_BUILD="${CUSTOM_TEST_ARTIFACT_BUILD_DIR}/custom-backend-build"
365+
# CUSTOM_BACKEND_TEST="$PWD/test/custom_backend"
366+
# python --version
367+
# mkdir -p "$CUSTOM_BACKEND_BUILD"
368+
# pushd "$CUSTOM_BACKEND_BUILD"
369+
# cmake "$CUSTOM_BACKEND_TEST" -DCMAKE_PREFIX_PATH="$SITE_PACKAGES" -DPython_EXECUTABLE="$(which python)" \
370+
# -DCMAKE_MODULE_PATH="$CUSTOM_TEST_MODULE_PATH" -DUSE_ROCM="$CUSTOM_TEST_USE_ROCM"
371+
# make VERBOSE=1
372+
# popd
373+
# assert_git_not_dirty
374374
else
375375
# Test no-Python build
376-
echo "Building libtorch"
377-
378-
# This is an attempt to mitigate flaky libtorch build OOM error. By default, the build parallelization
379-
# is set to be the number of CPU minus 2. So, let's try a more conservative value here. A 4xlarge has
380-
# 16 CPUs
381-
if [ -z "$MAX_JOBS_OVERRIDE" ]; then
382-
MAX_JOBS=$(nproc --ignore=4)
383-
export MAX_JOBS
384-
fi
385-
386-
# NB: Install outside of source directory (at the same level as the root
387-
# pytorch folder) so that it doesn't get cleaned away prior to docker push.
388-
BUILD_LIBTORCH_PY=$PWD/tools/build_libtorch.py
389-
mkdir -p ../cpp-build/caffe2
390-
pushd ../cpp-build/caffe2
391-
WERROR=1 VERBOSE=1 DEBUG=1 python "$BUILD_LIBTORCH_PY"
392-
popd
376+
# echo "Building libtorch"
377+
378+
# # This is an attempt to mitigate flaky libtorch build OOM error. By default, the build parallelization
379+
# # is set to be the number of CPU minus 2. So, let's try a more conservative value here. A 4xlarge has
380+
# # 16 CPUs
381+
# if [ -z "$MAX_JOBS_OVERRIDE" ]; then
382+
# MAX_JOBS=$(nproc --ignore=4)
383+
# export MAX_JOBS
384+
# fi
385+
386+
# # NB: Install outside of source directory (at the same level as the root
387+
# # pytorch folder) so that it doesn't get cleaned away prior to docker push.
388+
# BUILD_LIBTORCH_PY=$PWD/tools/build_libtorch.py
389+
# mkdir -p ../cpp-build/caffe2
390+
# pushd ../cpp-build/caffe2
391+
# WERROR=1 VERBOSE=1 DEBUG=1 python "$BUILD_LIBTORCH_PY"
392+
# popd
393393
fi
394394
fi
395395

396-
if [[ "$BUILD_ENVIRONMENT" != *libtorch* && "$BUILD_ENVIRONMENT" != *bazel* ]]; then
397-
# export test times so that potential sharded tests that'll branch off this build will use consistent data
398-
# don't do this for libtorch as libtorch is C++ only and thus won't have python tests run on its build
399-
python tools/stats/export_test_times.py
400-
fi
401-
# don't do this for bazel or s390x as they don't use sccache
402-
if [[ "$BUILD_ENVIRONMENT" != *s390x* && "$BUILD_ENVIRONMENT" != *ppc64le* && "$BUILD_ENVIRONMENT" != *-bazel-* ]]; then
403-
print_sccache_stats
404-
fi
396+
# if [[ "$BUILD_ENVIRONMENT" != *libtorch* && "$BUILD_ENVIRONMENT" != *bazel* ]]; then
397+
# # export test times so that potential sharded tests that'll branch off this build will use consistent data
398+
# # don't do this for libtorch as libtorch is C++ only and thus won't have python tests run on its build
399+
# python tools/stats/export_test_times.py
400+
# fi
401+
# # don't do this for bazel or s390x as they don't use sccache
402+
# if [[ "$BUILD_ENVIRONMENT" != *s390x* && "$BUILD_ENVIRONMENT" != *ppc64le* && "$BUILD_ENVIRONMENT" != *-bazel-* ]]; then
403+
# print_sccache_stats
404+
# fi

0 commit comments

Comments
 (0)