From 38bf48eb9eeeaf09bd5e9e57b97c62d97b309813 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Tue, 29 Oct 2024 15:15:43 -0700 Subject: [PATCH 1/9] Use pinned commit PyTorch on CI instead of nightly --- .ci/scripts/setup-linux.sh | 5 ++-- .ci/scripts/utils.sh | 7 +++-- install_requirements.py | 55 ++++++++++++++++++++++++++++---------- 3 files changed, 49 insertions(+), 18 deletions(-) diff --git a/.ci/scripts/setup-linux.sh b/.ci/scripts/setup-linux.sh index 5df4668f65c..f35bfa5a0db 100755 --- a/.ci/scripts/setup-linux.sh +++ b/.ci/scripts/setup-linux.sh @@ -19,6 +19,7 @@ else fi # As Linux job is running inside a Docker container, all of its dependencies -# have already been installed -install_executorch +# have already been installed, so we use PyTorch build from source here instead +# of nightly. This allows CI to test against latest commits from PyTorch +install_executorch "use-pt-pinned-commit" build_executorch_runner "${BUILD_TOOL}" diff --git a/.ci/scripts/utils.sh b/.ci/scripts/utils.sh index 64c512cdccd..f460b0e5474 100644 --- a/.ci/scripts/utils.sh +++ b/.ci/scripts/utils.sh @@ -20,8 +20,11 @@ install_executorch() { which pip # Install executorch, this assumes that Executorch is checked out in the # current directory. - # TODO(T199538337): clean up install scripts to use install_requirements.sh - ./install_requirements.sh --pybind xnnpack + if [[ $1 == "use-pt-pinned-commit" ]]; then + ./install_requirements.sh --pybind xnnpack --use-pt-pinned-commit + else + ./install_requirements.sh --pybind xnnpack + fi # Just print out the list of packages for debugging pip list } diff --git a/install_requirements.py b/install_requirements.py index dbdbecdaa3f..d3ac310f667 100644 --- a/install_requirements.py +++ b/install_requirements.py @@ -69,6 +69,7 @@ def python_is_compatible(): EXECUTORCH_BUILD_PYBIND = "OFF" CMAKE_ARGS = os.getenv("CMAKE_ARGS", "") CMAKE_BUILD_ARGS = os.getenv("CMAKE_BUILD_ARGS", "") +USE_PYTORCH_NIGHTLY = True for arg in sys.argv[1:]: if arg == "--pybind": @@ -90,6 +91,11 @@ def python_is_compatible(): shutil.rmtree(d, ignore_errors=True) print("Done cleaning build artifacts.") sys.exit(0) + elif arg == "--use-pt-pinned-commit": + # This option is used in CI to make sure that PyTorch build from the pinned commit + # is used instead of nightly. CI jobs wouldn't be able to catch regression from the + # latest PT commit otherwise + USE_PYTORCH_NIGHTLY = False else: print(f"Error: Unknown option {arg}") sys.exit(1) @@ -111,12 +117,22 @@ def python_is_compatible(): # The pip repository that hosts nightly torch packages. TORCH_NIGHTLY_URL = "https://download.pytorch.org/whl/nightly/cpu" -# pip packages needed by exir. -EXIR_REQUIREMENTS = [ - f"torch==2.6.0.{NIGHTLY_VERSION}", - f"torchvision==0.20.0.{NIGHTLY_VERSION}", # For testing. - "typing-extensions", -] +if USE_PYTORCH_NIGHTLY: + # pip packages needed by exir. + EXIR_REQUIREMENTS = [ + f"torch==2.6.0.{NIGHTLY_VERSION}", + f"torchvision==0.20.0.{NIGHTLY_VERSION}", # For testing. + "typing-extensions", + ] +else: + # This route is used in CI to test the pinned PyTorch commit. Note that we + # don't need to set any version number here because they have already been + # installed on CI before this step, so pip won't reinstall them + EXIR_REQUIREMENTS = [ + "torch", + "torchvision", + "typing-extensions", + ] # pip packages needed for development. DEVEL_REQUIREMENTS = [ @@ -129,14 +145,25 @@ def python_is_compatible(): "zstd", # Imported by resolve_buck.py. ] -# pip packages needed to run examples. -# TODO: Make each example publish its own requirements.txt -EXAMPLES_REQUIREMENTS = [ - "timm==1.0.7", - f"torchaudio==2.5.0.{NIGHTLY_VERSION}", - "torchsr==1.0.4", - "transformers==4.42.4", -] +if USE_PYTORCH_NIGHTLY: + # pip packages needed to run examples. + # TODO: Make each example publish its own requirements.txt + EXAMPLES_REQUIREMENTS = [ + "timm==1.0.7", + f"torchaudio==2.5.0.{NIGHTLY_VERSION}", + "torchsr==1.0.4", + "transformers==4.42.4", + ] +else: + # This route is used in CI to test the pinned PyTorch commit. Note that we + # don't need to set any version number here because they have already been + # installed on CI before this step, so pip won't reinstall them + EXAMPLES_REQUIREMENTS = [ + "timm==1.0.7", + "torchaudio", + "torchsr==1.0.4", + "transformers==4.42.4", + ] # Assemble the list of requirements to actually install. # TODO: Add options for reducing the number of requirements. From 364d3be191169a57a9ff89421a9c1b608b6041af Mon Sep 17 00:00:00 2001 From: Huy Do Date: Tue, 29 Oct 2024 15:37:09 -0700 Subject: [PATCH 2/9] Need to use a default value --- .ci/scripts/utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/scripts/utils.sh b/.ci/scripts/utils.sh index f460b0e5474..5bff362bcad 100644 --- a/.ci/scripts/utils.sh +++ b/.ci/scripts/utils.sh @@ -20,7 +20,7 @@ install_executorch() { which pip # Install executorch, this assumes that Executorch is checked out in the # current directory. - if [[ $1 == "use-pt-pinned-commit" ]]; then + if [[ "${1:-}" == "use-pt-pinned-commit" ]]; then ./install_requirements.sh --pybind xnnpack --use-pt-pinned-commit else ./install_requirements.sh --pybind xnnpack From d45886d20c86ec6bde725638cb2697c6b9372984 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Tue, 29 Oct 2024 18:39:05 -0700 Subject: [PATCH 3/9] Fix CI failures --- .ci/docker/common/install_base.sh | 3 ++- .ci/docker/requirements-ci.txt | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.ci/docker/common/install_base.sh b/.ci/docker/common/install_base.sh index cbca22cfa33..6ba645a3731 100755 --- a/.ci/docker/common/install_base.sh +++ b/.ci/docker/common/install_base.sh @@ -24,7 +24,8 @@ install_ubuntu() { gdb \ rsync \ libssl-dev \ - zip + zip \ + libomp5 # Cleanup package manager apt-get autoclean && apt-get clean diff --git a/.ci/docker/requirements-ci.txt b/.ci/docker/requirements-ci.txt index 52b2fd40060..c33cc533c02 100644 --- a/.ci/docker/requirements-ci.txt +++ b/.ci/docker/requirements-ci.txt @@ -1,5 +1,5 @@ mpmath==1.3.0 -numpy==1.22.0; python_version == '3.10' +numpy==1.21.3; python_version == '3.10' numpy==1.23.2; python_version == '3.11' numpy; python_version >= '3.12' PyYAML==6.0.1 From 3260158069bdc77589fe84cd76e1d90a681bfad0 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Tue, 29 Oct 2024 20:06:54 -0700 Subject: [PATCH 4/9] Update PT pin --- .ci/docker/ci_commit_pins/pytorch.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/docker/ci_commit_pins/pytorch.txt b/.ci/docker/ci_commit_pins/pytorch.txt index 0cfe7cae5a2..ec5fcbc90b7 100644 --- a/.ci/docker/ci_commit_pins/pytorch.txt +++ b/.ci/docker/ci_commit_pins/pytorch.txt @@ -1 +1 @@ -bd5482c7c3e1197e10c46ff739027f917d9c1fcc +23d590e518688f96e1d1947a08e9ca27df3e67e4 From a4c318868963677ae2a8209d20b389430d1d494e Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 30 Oct 2024 13:39:11 -0700 Subject: [PATCH 5/9] Fix libstdc++ installation --- .ci/docker/common/install_base.sh | 3 +-- .ci/docker/common/install_clang.sh | 2 +- .ci/scripts/setup-qnn-deps.sh | 6 ++++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.ci/docker/common/install_base.sh b/.ci/docker/common/install_base.sh index 6ba645a3731..cbca22cfa33 100755 --- a/.ci/docker/common/install_base.sh +++ b/.ci/docker/common/install_base.sh @@ -24,8 +24,7 @@ install_ubuntu() { gdb \ rsync \ libssl-dev \ - zip \ - libomp5 + zip # Cleanup package manager apt-get autoclean && apt-get clean diff --git a/.ci/docker/common/install_clang.sh b/.ci/docker/common/install_clang.sh index ea0848a8d31..f3c479d058e 100755 --- a/.ci/docker/common/install_clang.sh +++ b/.ci/docker/common/install_clang.sh @@ -13,7 +13,7 @@ install_ubuntu() { apt-get install -y --no-install-recommends clang-"$CLANG_VERSION" apt-get install -y --no-install-recommends llvm-"$CLANG_VERSION" # Also require LLD linker from llvm and libomp to build PyTorch from source - apt-get install -y lld "libomp-${CLANG_VERSION}-dev" + apt-get install -y lld "libomp-${CLANG_VERSION}-dev" "libc++-${CLANG_VERSION}-dev" # Use update-alternatives to make this version the default update-alternatives --install /usr/bin/clang clang /usr/bin/clang-"$CLANG_VERSION" 50 diff --git a/.ci/scripts/setup-qnn-deps.sh b/.ci/scripts/setup-qnn-deps.sh index 92ffd07bccc..12809748129 100644 --- a/.ci/scripts/setup-qnn-deps.sh +++ b/.ci/scripts/setup-qnn-deps.sh @@ -31,8 +31,9 @@ install_qnn() { } setup_libc++() { + clang_version=$1 sudo apt-get update - pkgs_to_check=('libc++-dev') + pkgs_to_check=("libc++-${clang_version}-dev") j=0 while [ $j -lt ${#pkgs_to_check[*]} ]; do install_status=$(verify_pkg_installed ${pkgs_to_check[$j]}) @@ -47,5 +48,6 @@ setup_libc++() { done } -setup_libc++ +# This needs to match with the clang version from the Docker image +setup_libc++ 12 install_qnn From 6a1753eca21411f3e11f6c837b407414ef44cc51 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 30 Oct 2024 14:43:23 -0700 Subject: [PATCH 6/9] Use a higher timeout for some jobs --- .github/workflows/trunk.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml index 2d4bb8184b8..9c240533b5e 100644 --- a/.github/workflows/trunk.yml +++ b/.github/workflows/trunk.yml @@ -137,6 +137,7 @@ jobs: docker-image: executorch-ubuntu-22.04-arm-sdk submodules: 'true' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 90 script: | # The generic Linux job chooses to use base env, not the one setup by the image CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") @@ -162,6 +163,7 @@ jobs: docker-image: executorch-ubuntu-22.04-arm-sdk submodules: 'true' ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + timeout: 90 script: | # The generic Linux job chooses to use base env, not the one setup by the image CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]") From 2a8b480cf282cd3b816ba4acc34b275b65a10c34 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Thu, 31 Oct 2024 17:28:13 -0700 Subject: [PATCH 7/9] Fix review comments --- install_requirements.py | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/install_requirements.py b/install_requirements.py index d3ac310f667..bb96cec4dc3 100644 --- a/install_requirements.py +++ b/install_requirements.py @@ -124,6 +124,15 @@ def python_is_compatible(): f"torchvision==0.20.0.{NIGHTLY_VERSION}", # For testing. "typing-extensions", ] + + # pip packages needed to run examples. + # TODO: Make each example publish its own requirements.txt + EXAMPLES_REQUIREMENTS = [ + "timm==1.0.7", + f"torchaudio==2.5.0.{NIGHTLY_VERSION}", + "torchsr==1.0.4", + "transformers==4.42.4", + ] else: # This route is used in CI to test the pinned PyTorch commit. Note that we # don't need to set any version number here because they have already been @@ -134,6 +143,13 @@ def python_is_compatible(): "typing-extensions", ] + EXAMPLES_REQUIREMENTS = [ + "timm==1.0.7", + "torchaudio", + "torchsr==1.0.4", + "transformers==4.42.4", + ] + # pip packages needed for development. DEVEL_REQUIREMENTS = [ "cmake", # For building binary targets. @@ -145,26 +161,6 @@ def python_is_compatible(): "zstd", # Imported by resolve_buck.py. ] -if USE_PYTORCH_NIGHTLY: - # pip packages needed to run examples. - # TODO: Make each example publish its own requirements.txt - EXAMPLES_REQUIREMENTS = [ - "timm==1.0.7", - f"torchaudio==2.5.0.{NIGHTLY_VERSION}", - "torchsr==1.0.4", - "transformers==4.42.4", - ] -else: - # This route is used in CI to test the pinned PyTorch commit. Note that we - # don't need to set any version number here because they have already been - # installed on CI before this step, so pip won't reinstall them - EXAMPLES_REQUIREMENTS = [ - "timm==1.0.7", - "torchaudio", - "torchsr==1.0.4", - "transformers==4.42.4", - ] - # Assemble the list of requirements to actually install. # TODO: Add options for reducing the number of requirements. REQUIREMENTS_TO_INSTALL = EXIR_REQUIREMENTS + DEVEL_REQUIREMENTS + EXAMPLES_REQUIREMENTS From 3d41e75f79d3b3a63758a72fbf0628c21aca9c34 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Thu, 31 Oct 2024 17:57:25 -0700 Subject: [PATCH 8/9] Minor format fix --- install_requirements.py | 51 +++++++++++++++-------------------------- 1 file changed, 19 insertions(+), 32 deletions(-) diff --git a/install_requirements.py b/install_requirements.py index bb96cec4dc3..8c967da1e86 100644 --- a/install_requirements.py +++ b/install_requirements.py @@ -117,38 +117,25 @@ def python_is_compatible(): # The pip repository that hosts nightly torch packages. TORCH_NIGHTLY_URL = "https://download.pytorch.org/whl/nightly/cpu" -if USE_PYTORCH_NIGHTLY: - # pip packages needed by exir. - EXIR_REQUIREMENTS = [ - f"torch==2.6.0.{NIGHTLY_VERSION}", - f"torchvision==0.20.0.{NIGHTLY_VERSION}", # For testing. - "typing-extensions", - ] - - # pip packages needed to run examples. - # TODO: Make each example publish its own requirements.txt - EXAMPLES_REQUIREMENTS = [ - "timm==1.0.7", - f"torchaudio==2.5.0.{NIGHTLY_VERSION}", - "torchsr==1.0.4", - "transformers==4.42.4", - ] -else: - # This route is used in CI to test the pinned PyTorch commit. Note that we - # don't need to set any version number here because they have already been - # installed on CI before this step, so pip won't reinstall them - EXIR_REQUIREMENTS = [ - "torch", - "torchvision", - "typing-extensions", - ] - - EXAMPLES_REQUIREMENTS = [ - "timm==1.0.7", - "torchaudio", - "torchsr==1.0.4", - "transformers==4.42.4", - ] +# pip packages needed by exir. +EXIR_REQUIREMENTS = [ + f"torch==2.6.0.{NIGHTLY_VERSION}" if USE_PYTORCH_NIGHTLY else "torch", + ( + f"torchvision==0.20.0.{NIGHTLY_VERSION}" + if USE_PYTORCH_NIGHTLY + else "torchvision" + ), # For testing. + "typing-extensions", +] + +# pip packages needed to run examples. +# TODO: Make each example publish its own requirements.txt +EXAMPLES_REQUIREMENTS = [ + "timm==1.0.7", + f"torchaudio==2.5.0.{NIGHTLY_VERSION}" if USE_PYTORCH_NIGHTLY else "torchaudio", + "torchsr==1.0.4", + "transformers==4.42.4", +] # pip packages needed for development. DEVEL_REQUIREMENTS = [ From f3ce7efdd0dd2acd1c0ae15856b13680e6601443 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Thu, 31 Oct 2024 18:28:34 -0700 Subject: [PATCH 9/9] Add back the comment --- install_requirements.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/install_requirements.py b/install_requirements.py index 8c967da1e86..4c3cbc2c45b 100644 --- a/install_requirements.py +++ b/install_requirements.py @@ -119,6 +119,9 @@ def python_is_compatible(): # pip packages needed by exir. EXIR_REQUIREMENTS = [ + # Setting USE_PYTORCH_NIGHTLY to false to test the pinned PyTorch commit. Note + # that we don't need to set any version number there because they have already + # been installed on CI before this step, so pip won't reinstall them f"torch==2.6.0.{NIGHTLY_VERSION}" if USE_PYTORCH_NIGHTLY else "torch", ( f"torchvision==0.20.0.{NIGHTLY_VERSION}"