From 9e0e2594861b9d8991099d070799a7bcc10a3432 Mon Sep 17 00:00:00 2001 From: Mergen Nachin Date: Fri, 13 Jun 2025 17:49:34 -0400 Subject: [PATCH] Install examples and domain libraries as last (optional) step --- install_executorch.py | 25 ++++++++++--- install_requirements.py | 70 +++++++++++++++++++++++++++---------- requirements-dev.txt | 4 +++ requirements-lintrunner.txt | 4 --- 4 files changed, 76 insertions(+), 27 deletions(-) diff --git a/install_executorch.py b/install_executorch.py index fa34092aceb..12452ef4994 100644 --- a/install_executorch.py +++ b/install_executorch.py @@ -16,6 +16,7 @@ from contextlib import contextmanager from install_requirements import ( + install_optional_example_requirements, install_requirements, python_is_compatible, TORCH_NIGHTLY_URL, @@ -157,6 +158,13 @@ def _parse_args() -> argparse.Namespace: "picked up without rebuilding the wheel. Extension libraries will be " "installed inside the source tree.", ) + parser.add_argument( + "--minimal", + "-m", + action="store_true", + help="Only installs necessary dependencies for core executorch and skips " + " packages necessary for running example scripts.", + ) return parser.parse_args() @@ -182,9 +190,13 @@ def main(args): # This option is used in CI to make sure that PyTorch build from the pinned commit # is used instead of nightly. CI jobs wouldn't be able to catch regression from the # latest PT commit otherwise - install_requirements(use_pytorch_nightly=not args.use_pt_pinned_commit) - os.execvp( - sys.executable, + use_pytorch_nightly = not args.use_pt_pinned_commit + + # Step 1: Install core dependencies first + install_requirements(use_pytorch_nightly) + + # Step 2: Install core package + cmd = ( [ sys.executable, "-m", @@ -198,8 +210,13 @@ def main(args): "-v", "--extra-index-url", TORCH_NIGHTLY_URL, - ], + ] ) + subprocess.run(cmd, check=True) + + # Step 3: Extra (optional) packages that is only useful for running examples. + if not args.minimal: + install_optional_example_requirements(use_pytorch_nightly) if __name__ == "__main__": diff --git a/install_requirements.py b/install_requirements.py index f60020dbbbf..25abc614b0a 100644 --- a/install_requirements.py +++ b/install_requirements.py @@ -76,27 +76,15 @@ def python_is_compatible(): def install_requirements(use_pytorch_nightly): # pip packages needed by exir. - EXIR_REQUIREMENTS = [ + TORCH_PACKAGE = [ # Setting use_pytorch_nightly to false to test the pinned PyTorch commit. Note # that we don't need to set any version number there because they have already # been installed on CI before this step, so pip won't reinstall them f"torch==2.8.0.{NIGHTLY_VERSION}" if use_pytorch_nightly else "torch", - ( - f"torchvision==0.23.0.{NIGHTLY_VERSION}" - if use_pytorch_nightly - else "torchvision" - ), # For testing. ] - EXAMPLES_REQUIREMENTS = [ - f"torchaudio==2.8.0.{NIGHTLY_VERSION}" if use_pytorch_nightly else "torchaudio", - ] - - # Assemble the list of requirements to actually install. - # TODO: Add options for reducing the number of requirements. - REQUIREMENTS_TO_INSTALL = EXIR_REQUIREMENTS + EXAMPLES_REQUIREMENTS - - # Install the requirements. `--extra-index-url` tells pip to look for package + # Install the requirements for core ExecuTorch package. + # `--extra-index-url` tells pip to look for package # versions on the provided URL if they aren't available on the default URL. subprocess.run( [ @@ -105,10 +93,8 @@ def install_requirements(use_pytorch_nightly): "pip", "install", "-r", - "requirements-examples.txt", - "-r", "requirements-dev.txt", - *REQUIREMENTS_TO_INSTALL, + *TORCH_PACKAGE, "--extra-index-url", TORCH_NIGHTLY_URL, ], @@ -139,6 +125,44 @@ def install_requirements(use_pytorch_nightly): ) +def install_optional_example_requirements(use_pytorch_nightly): + print("Installing packages in requirements-examples.txt") + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + "-r", + "requirements-examples.txt", + ], + check=True, + ) + + print("Installing torch domain libraries") + DOMAIN_LIBRARIES = [ + ( + f"torchvision==0.23.0.{NIGHTLY_VERSION}" + if use_pytorch_nightly + else "torchvision" + ), + f"torchaudio==2.8.0.{NIGHTLY_VERSION}" if use_pytorch_nightly else "torchaudio", + ] + # Then install domain libraries + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + *DOMAIN_LIBRARIES, + "--extra-index-url", + TORCH_NIGHTLY_URL, + ], + check=True, + ) + + def main(args): parser = argparse.ArgumentParser() parser.add_argument( @@ -146,8 +170,16 @@ def main(args): action="store_true", help="build from the pinned PyTorch commit instead of nightly", ) + parser.add_argument( + "--example", + action="store_true", + help="Also installs required packages for running example scripts.", + ) args = parser.parse_args(args) - install_requirements(use_pytorch_nightly=not bool(args.use_pt_pinned_commit)) + use_pytorch_nightly=not bool(args.use_pt_pinned_commit) + install_requirements(use_pytorch_nightly) + if args.example: + install_optional_example_requirements(use_pytorch_nightly) if __name__ == "__main__": diff --git a/requirements-dev.txt b/requirements-dev.txt index a05cf6fa149..a4ed212fb65 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,5 @@ +# Pip packages needed to build from source. Mainly for development of ExecuTorch. + cmake>=3.19, <4.0.0 # For building binary targets in the wheel. pip>=23 # For building the pip package. pyyaml # Imported by the kernel codegen tools. @@ -5,3 +7,5 @@ setuptools>=63 # For building the pip package contents. tomli # Imported by extract_sources.py when using python < 3.11. wheel # For building the pip package archive. zstd # Imported by resolve_buck.py. +lintrunner==0.12.7 +lintrunner-adapters==0.12.4 diff --git a/requirements-lintrunner.txt b/requirements-lintrunner.txt index 53d7b91bb54..02b8ab67051 100644 --- a/requirements-lintrunner.txt +++ b/requirements-lintrunner.txt @@ -1,7 +1,3 @@ -# Lintrunner itself -lintrunner==0.12.7 -lintrunner-adapters==0.12.4 - # Flake 8 and its dependencies flake8==6.1.0 flake8-breakpoint==1.1.0