diff --git a/.ci/docker/ci_commit_pins/pytorch.txt b/.ci/docker/ci_commit_pins/pytorch.txt index 0cfe7cae5a2..ec5fcbc90b7 100644 --- a/.ci/docker/ci_commit_pins/pytorch.txt +++ b/.ci/docker/ci_commit_pins/pytorch.txt @@ -1 +1 @@ -bd5482c7c3e1197e10c46ff739027f917d9c1fcc +23d590e518688f96e1d1947a08e9ca27df3e67e4 diff --git a/.ci/docker/common/install_clang.sh b/.ci/docker/common/install_clang.sh index ea0848a8d31..f3c479d058e 100755 --- a/.ci/docker/common/install_clang.sh +++ b/.ci/docker/common/install_clang.sh @@ -13,7 +13,7 @@ install_ubuntu() { apt-get install -y --no-install-recommends clang-"$CLANG_VERSION" apt-get install -y --no-install-recommends llvm-"$CLANG_VERSION" # Also require LLD linker from llvm and libomp to build PyTorch from source - apt-get install -y lld "libomp-${CLANG_VERSION}-dev" + apt-get install -y lld "libomp-${CLANG_VERSION}-dev" "libc++-${CLANG_VERSION}-dev" # Use update-alternatives to make this version the default update-alternatives --install /usr/bin/clang clang /usr/bin/clang-"$CLANG_VERSION" 50 diff --git a/.ci/docker/requirements-ci.txt b/.ci/docker/requirements-ci.txt index 52b2fd40060..c33cc533c02 100644 --- a/.ci/docker/requirements-ci.txt +++ b/.ci/docker/requirements-ci.txt @@ -1,5 +1,5 @@ mpmath==1.3.0 -numpy==1.22.0; python_version == '3.10' +numpy==1.21.3; python_version == '3.10' numpy==1.23.2; python_version == '3.11' numpy; python_version >= '3.12' PyYAML==6.0.1 diff --git a/.ci/scripts/setup-linux.sh b/.ci/scripts/setup-linux.sh index 5df4668f65c..f35bfa5a0db 100755 --- a/.ci/scripts/setup-linux.sh +++ b/.ci/scripts/setup-linux.sh @@ -19,6 +19,7 @@ else fi # As Linux job is running inside a Docker container, all of its dependencies -# have already been installed -install_executorch +# have already been installed, so we use PyTorch build from source here instead +# of nightly. This allows CI to test against latest commits from PyTorch +install_executorch "use-pt-pinned-commit" build_executorch_runner "${BUILD_TOOL}" diff --git a/.ci/scripts/setup-qnn-deps.sh b/.ci/scripts/setup-qnn-deps.sh index 92ffd07bccc..12809748129 100644 --- a/.ci/scripts/setup-qnn-deps.sh +++ b/.ci/scripts/setup-qnn-deps.sh @@ -31,8 +31,9 @@ install_qnn() { } setup_libc++() { + clang_version=$1 sudo apt-get update - pkgs_to_check=('libc++-dev') + pkgs_to_check=("libc++-${clang_version}-dev") j=0 while [ $j -lt ${#pkgs_to_check[*]} ]; do install_status=$(verify_pkg_installed ${pkgs_to_check[$j]}) @@ -47,5 +48,6 @@ setup_libc++() { done } -setup_libc++ +# This needs to match with the clang version from the Docker image +setup_libc++ 12 install_qnn diff --git a/.ci/scripts/utils.sh b/.ci/scripts/utils.sh index 64c512cdccd..5bff362bcad 100644 --- a/.ci/scripts/utils.sh +++ b/.ci/scripts/utils.sh @@ -20,8 +20,11 @@ install_executorch() { which pip # Install executorch, this assumes that Executorch is checked out in the # current directory. - # TODO(T199538337): clean up install scripts to use install_requirements.sh - ./install_requirements.sh --pybind xnnpack + if [[ "${1:-}" == "use-pt-pinned-commit" ]]; then + ./install_requirements.sh --pybind xnnpack --use-pt-pinned-commit + else + ./install_requirements.sh --pybind xnnpack + fi # Just print out the list of packages for debugging pip list } diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml index 2d4bb8184b8..9c240533b5e 100644 --- a/.github/workflows/trunk.yml +++ b/.github/workflows/trunk.yml @@ -137,6 +137,7 @@ jobs: docker-image: executorch-ubuntu-22.04-arm-sdk submodules: 'true' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 90 script: | # The generic Linux job chooses to use base env, not the one setup by the image CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") @@ -162,6 +163,7 @@ jobs: docker-image: executorch-ubuntu-22.04-arm-sdk submodules: 'true' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 90 script: | # The generic Linux job chooses to use base env, not the one setup by the image CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") diff --git a/install_requirements.py b/install_requirements.py index dbdbecdaa3f..4c3cbc2c45b 100644 --- a/install_requirements.py +++ b/install_requirements.py @@ -69,6 +69,7 @@ def python_is_compatible(): EXECUTORCH_BUILD_PYBIND = "OFF" CMAKE_ARGS = os.getenv("CMAKE_ARGS", "") CMAKE_BUILD_ARGS = os.getenv("CMAKE_BUILD_ARGS", "") +USE_PYTORCH_NIGHTLY = True for arg in sys.argv[1:]: if arg == "--pybind": @@ -90,6 +91,11 @@ def python_is_compatible(): shutil.rmtree(d, ignore_errors=True) print("Done cleaning build artifacts.") sys.exit(0) + elif arg == "--use-pt-pinned-commit": + # This option is used in CI to make sure that PyTorch build from the pinned commit + # is used instead of nightly. CI jobs wouldn't be able to catch regression from the + # latest PT commit otherwise + USE_PYTORCH_NIGHTLY = False else: print(f"Error: Unknown option {arg}") sys.exit(1) @@ -113,11 +119,27 @@ def python_is_compatible(): # pip packages needed by exir. EXIR_REQUIREMENTS = [ - f"torch==2.6.0.{NIGHTLY_VERSION}", - f"torchvision==0.20.0.{NIGHTLY_VERSION}", # For testing. + # Setting USE_PYTORCH_NIGHTLY to false to test the pinned PyTorch commit. Note + # that we don't need to set any version number there because they have already + # been installed on CI before this step, so pip won't reinstall them + f"torch==2.6.0.{NIGHTLY_VERSION}" if USE_PYTORCH_NIGHTLY else "torch", + ( + f"torchvision==0.20.0.{NIGHTLY_VERSION}" + if USE_PYTORCH_NIGHTLY + else "torchvision" + ), # For testing. "typing-extensions", ] +# pip packages needed to run examples. +# TODO: Make each example publish its own requirements.txt +EXAMPLES_REQUIREMENTS = [ + "timm==1.0.7", + f"torchaudio==2.5.0.{NIGHTLY_VERSION}" if USE_PYTORCH_NIGHTLY else "torchaudio", + "torchsr==1.0.4", + "transformers==4.42.4", +] + # pip packages needed for development. DEVEL_REQUIREMENTS = [ "cmake", # For building binary targets. @@ -129,15 +151,6 @@ def python_is_compatible(): "zstd", # Imported by resolve_buck.py. ] -# pip packages needed to run examples. -# TODO: Make each example publish its own requirements.txt -EXAMPLES_REQUIREMENTS = [ - "timm==1.0.7", - f"torchaudio==2.5.0.{NIGHTLY_VERSION}", - "torchsr==1.0.4", - "transformers==4.42.4", -] - # Assemble the list of requirements to actually install. # TODO: Add options for reducing the number of requirements. REQUIREMENTS_TO_INSTALL = EXIR_REQUIREMENTS + DEVEL_REQUIREMENTS + EXAMPLES_REQUIREMENTS