From 6f84f98d46d9e502dcf5ea4dd100672d71b05553 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sun, 13 Apr 2025 23:59:23 -0700 Subject: [PATCH 1/3] Clone submodules recursively in install_executorch.py Somehow we still don't clone submodules recursively for some submodules, like tokenizers --- install_executorch.py | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/install_executorch.py b/install_executorch.py index 1d3fe8af1fb..53e3e5c3040 100644 --- a/install_executorch.py +++ b/install_executorch.py @@ -17,11 +17,7 @@ from contextlib import contextmanager from typing import List, Tuple -from install_requirements import ( - install_requirements, - python_is_compatible, - TORCH_NIGHTLY_URL, -) +from install_requirements import install_requirements, python_is_compatible, TORCH_URL # Set up logging logging.basicConfig( @@ -53,7 +49,7 @@ def clean(): # Please keep this insync with `ShouldBuild.pybindings` in setup.py. -VALID_PYBINDS = ["coreml", "mps", "xnnpack", "training", "openvino"] +VALID_PYBINDS = ["coreml", "mps", "xnnpack", "training"] ################################################################################ @@ -120,8 +116,8 @@ def check_folder(folder: str, file: str) -> bool: if missing_submodules: logger.warning("Some required submodules are missing. Updating submodules...") try: - subprocess.check_call(["git", "submodule", "sync"]) - subprocess.check_call(["git", "submodule", "update", "--init"]) + subprocess.check_call(["git", "submodule", "sync", "--recursive"]) + subprocess.check_call(["git", "submodule", "update", "--init", "--recursive"]) except subprocess.CalledProcessError as e: logger.error(f"Error updating submodules: {e}") exit(1) @@ -130,13 +126,8 @@ def check_folder(folder: str, file: str) -> bool: for path, file in missing_submodules.items(): if not check_folder(path, file): logger.error(f"{file} not found in {path}.") - logger.error("Please run `git submodule update --init`.") + logger.error("Submodule update failed. Please run `git submodule update --init --recursive` manually.") exit(1) - # Go into tokenizers submodule and install its submodules - tokenizers_path = get_required_submodule_paths().get("tokenizers", None) - if tokenizers_path: - with pushd(tokenizers_path): - subprocess.check_call(["git", "submodule", "update", "--init"]) logger.info("All required submodules are present.") @@ -207,10 +198,14 @@ def main(args): use_pytorch_nightly = True wants_pybindings_off, pybind_defines = _list_pybind_defines(args) - if wants_pybindings_off: - cmake_args.append("-DEXECUTORCH_BUILD_PYBIND=OFF") - else: - cmake_args += pybind_defines + if not wants_pybindings_off: + if len(pybind_defines) > 0: + # If the user explicitly provides a list of bindings, just use them + cmake_args += pybind_defines + else: + # If the user has not set pybindings off but also has not provided + # a list, then turn on xnnpack by default + cmake_args.append("-DEXECUTORCH_BUILD_XNNPACK=ON") if args.clean: clean() @@ -256,7 +251,7 @@ def main(args): "--no-build-isolation", "-v", "--extra-index-url", - TORCH_NIGHTLY_URL, + TORCH_URL, ], check=True, ) From 051387e7b281a282b2b96dc7ab2ff804a03295d0 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Mon, 14 Apr 2025 00:01:54 -0700 Subject: [PATCH 2/3] Update install_executorch.py --- install_executorch.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/install_executorch.py b/install_executorch.py index 53e3e5c3040..3f901869c3d 100644 --- a/install_executorch.py +++ b/install_executorch.py @@ -17,7 +17,11 @@ from contextlib import contextmanager from typing import List, Tuple -from install_requirements import install_requirements, python_is_compatible, TORCH_URL +from install_requirements import ( + install_requirements, + python_is_compatible, + TORCH_NIGHTLY_URL, +) # Set up logging logging.basicConfig( @@ -49,7 +53,7 @@ def clean(): # Please keep this insync with `ShouldBuild.pybindings` in setup.py. -VALID_PYBINDS = ["coreml", "mps", "xnnpack", "training"] +VALID_PYBINDS = ["coreml", "mps", "xnnpack", "training", "openvino"] ################################################################################ @@ -198,14 +202,10 @@ def main(args): use_pytorch_nightly = True wants_pybindings_off, pybind_defines = _list_pybind_defines(args) - if not wants_pybindings_off: - if len(pybind_defines) > 0: - # If the user explicitly provides a list of bindings, just use them - cmake_args += pybind_defines - else: - # If the user has not set pybindings off but also has not provided - # a list, then turn on xnnpack by default - cmake_args.append("-DEXECUTORCH_BUILD_XNNPACK=ON") + if wants_pybindings_off: + cmake_args.append("-DEXECUTORCH_BUILD_PYBIND=OFF") + else: + cmake_args += pybind_defines if args.clean: clean() @@ -251,7 +251,7 @@ def main(args): "--no-build-isolation", "-v", "--extra-index-url", - TORCH_URL, + TORCH_NIGHTLY_URL, ], check=True, ) From f1d0b8238402cfbdf3531dcb4f797d3530a78d8e Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Mon, 14 Apr 2025 10:20:56 -0700 Subject: [PATCH 3/3] Update install_executorch.py --- install_executorch.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/install_executorch.py b/install_executorch.py index 3f901869c3d..4c7b51ef239 100644 --- a/install_executorch.py +++ b/install_executorch.py @@ -121,7 +121,9 @@ def check_folder(folder: str, file: str) -> bool: logger.warning("Some required submodules are missing. Updating submodules...") try: subprocess.check_call(["git", "submodule", "sync", "--recursive"]) - subprocess.check_call(["git", "submodule", "update", "--init", "--recursive"]) + subprocess.check_call( + ["git", "submodule", "update", "--init", "--recursive"] + ) except subprocess.CalledProcessError as e: logger.error(f"Error updating submodules: {e}") exit(1) @@ -130,7 +132,9 @@ def check_folder(folder: str, file: str) -> bool: for path, file in missing_submodules.items(): if not check_folder(path, file): logger.error(f"{file} not found in {path}.") - logger.error("Submodule update failed. Please run `git submodule update --init --recursive` manually.") + logger.error( + "Submodule update failed. Please run `git submodule update --init --recursive` manually." + ) exit(1) logger.info("All required submodules are present.")