Skip to content

Commit 2c8e706

Browse files
larryliu0820larryliu0820
authored
[BE] Move submodule check out from setup.py (#8315)
* [BE] Move submodule check out from setup.py Summary: Per comment in #8203: #8203 (comment) Moving the logic of checking submodule and update out from setup.py and into install_executorch.py. Test Plan: Reviewers: Subscribers: Tasks: Tags: * lint Summary: Test Plan: Reviewers: Subscribers: Tasks: Tags: * Move submodule handling to before install requirements Summary: Test Plan: Reviewers: Subscribers: Tasks: Tags:
1 parent 16281ce commit 2c8e706

File tree

2 files changed

+117
-124
lines changed

2 files changed

+117
-124
lines changed

‎install_executorch.py‎

Lines changed: 89 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import argparse
1010
import glob
1111
import itertools
12+
import logging
1213
import os
1314
import shutil
1415
import subprocess
@@ -20,6 +21,12 @@
2021
TORCH_NIGHTLY_URL,
2122
)
2223

24+
# Set up logging
25+
logging.basicConfig(
26+
level=logging.INFO, format="%(asctime)s [ExecuTorch] %(levelname)s: %(message)s"
27+
)
28+
logger = logging.getLogger()
29+
2330

2431
def clean():
2532
print("Cleaning build artifacts...")
@@ -35,6 +42,83 @@ def clean():
3542
VALID_PYBINDS = ["coreml", "mps", "xnnpack", "training"]
3643

3744

45+
################################################################################
46+
# Git submodules
47+
################################################################################
48+
# The following submodules are required to be able to build ExecuTorch. If any of
49+
# these folders are missing or missing CMakeLists.txt, we will run
50+
# `git submodule update` to try to fix it. If the command fails, we will raise an
51+
# error.
52+
# An alternative to this would be to run `git submodule status` and run
53+
# `git submodule update` if there's any local changes. However this is a bit
54+
# too restrictive for users who modifies and tests the dependencies locally.
55+
56+
# keep sorted
57+
REQUIRED_SUBMODULES = {
58+
"ao": "LICENSE", # No CMakeLists.txt, choose a sort of stable file to check.
59+
"cpuinfo": "CMakeLists.txt",
60+
"eigen": "CMakeLists.txt",
61+
"flatbuffers": "CMakeLists.txt",
62+
"FP16": "CMakeLists.txt",
63+
"FXdiv": "CMakeLists.txt",
64+
"gflags": "CMakeLists.txt",
65+
"prelude": "BUCK",
66+
"pthreadpool": "CMakeLists.txt",
67+
"pybind11": "CMakeLists.txt",
68+
"XNNPACK": "CMakeLists.txt",
69+
}
70+
71+
72+
def get_required_submodule_paths():
73+
gitmodules_path = os.path.join(os.getcwd(), ".gitmodules")
74+
75+
if not os.path.isfile(gitmodules_path):
76+
logger.error(".gitmodules file not found.")
77+
exit(1)
78+
79+
with open(gitmodules_path, "r") as file:
80+
lines = file.readlines()
81+
82+
# Extract paths of required submodules
83+
required_paths = {}
84+
for line in lines:
85+
if line.strip().startswith("path ="):
86+
path = line.split("=")[1].strip()
87+
for submodule, file_name in REQUIRED_SUBMODULES.items():
88+
if submodule in path:
89+
required_paths[path] = file_name
90+
return required_paths
91+
92+
93+
def check_and_update_submodules():
94+
def check_folder(folder: str, file: str) -> bool:
95+
return os.path.isdir(folder) and os.path.isfile(os.path.join(folder, file))
96+
97+
# Check if the directories exist for each required submodule
98+
missing_submodules = {}
99+
for path, file in get_required_submodule_paths().items():
100+
if not check_folder(path, file):
101+
missing_submodules[path] = file
102+
103+
# If any required submodule directories are missing, update them
104+
if missing_submodules:
105+
logger.warning("Some required submodules are missing. Updating submodules...")
106+
try:
107+
subprocess.check_call(["git", "submodule", "sync"])
108+
subprocess.check_call(["git", "submodule", "update", "--init"])
109+
except subprocess.CalledProcessError as e:
110+
logger.error(f"Error updating submodules: {e}")
111+
exit(1)
112+
113+
# After updating submodules, check again
114+
for path, file in missing_submodules.items():
115+
if not check_folder(path, file):
116+
logger.error(f"{file} not found in {path}.")
117+
logger.error("Please run `git submodule update --init`.")
118+
exit(1)
119+
logger.info("All required submodules are present.")
120+
121+
38122
def main(args):
39123
if not python_is_compatible():
40124
sys.exit(1)
@@ -95,8 +179,6 @@ def main(args):
95179
# latest PT commit otherwise
96180
use_pytorch_nightly = False
97181

98-
install_requirements(use_pytorch_nightly)
99-
100182
# If --pybind is not set explicitly for backends (e.g., --pybind xnnpack)
101183
# or is not turned off explicitly (--pybind off)
102184
# then install XNNPACK by default.
@@ -120,6 +202,11 @@ def main(args):
120202
os.environ["EXECUTORCH_BUILD_PYBIND"] = EXECUTORCH_BUILD_PYBIND
121203
os.environ["CMAKE_ARGS"] = CMAKE_ARGS
122204

205+
# Check if the required submodules are present and update them if not
206+
check_and_update_submodules()
207+
208+
install_requirements(use_pytorch_nightly)
209+
123210
# Run the pip install command
124211
subprocess.run(
125212
[

‎setup.py‎

Lines changed: 28 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,6 @@
5555
# Import this before distutils so that setuptools can intercept the distuils
5656
# imports.
5757
import setuptools # noqa: F401 # usort: skip
58-
import logging
5958
import subprocess
6059

6160
from distutils import log
@@ -68,91 +67,6 @@
6867
from setuptools.command.build_ext import build_ext
6968
from setuptools.command.build_py import build_py
7069

71-
# Set up logging
72-
logging.basicConfig(
73-
level=logging.INFO, format="%(asctime)s [ExecuTorch] %(levelname)s: %(message)s"
74-
)
75-
logger = logging.getLogger()
76-
77-
# For information on setuptools Command subclassing see
78-
# https://setuptools.pypa.io/en/latest/userguide/extension.html
79-
80-
################################################################################
81-
# Git submodules
82-
################################################################################
83-
# The following submodules are required to be able to build ExecuTorch. If any of
84-
# these folders are missing or missing CMakeLists.txt, we will run
85-
# `git submodule update` to try to fix it. If the command fails, we will raise an
86-
# error.
87-
# An alternative to this would be to run `git submodule status` and run
88-
# `git submodule update` if there's any local changes. However this is a bit
89-
# too restrictive for users who modifies and tests the dependencies locally.
90-
91-
# keep sorted
92-
REQUIRED_SUBMODULES = {
93-
"ao": "LICENSE", # No CMakeLists.txt, choose a sort of stable file to check.
94-
"cpuinfo": "CMakeLists.txt",
95-
"eigen": "CMakeLists.txt",
96-
"flatbuffers": "CMakeLists.txt",
97-
"FP16": "CMakeLists.txt",
98-
"FXdiv": "CMakeLists.txt",
99-
"gflags": "CMakeLists.txt",
100-
"prelude": "BUCK",
101-
"pthreadpool": "CMakeLists.txt",
102-
"pybind11": "CMakeLists.txt",
103-
"XNNPACK": "CMakeLists.txt",
104-
}
105-
106-
107-
def get_required_submodule_paths():
108-
gitmodules_path = os.path.join(os.getcwd(), ".gitmodules")
109-
110-
if not os.path.isfile(gitmodules_path):
111-
logger.error(".gitmodules file not found.")
112-
exit(1)
113-
114-
with open(gitmodules_path, "r") as file:
115-
lines = file.readlines()
116-
117-
# Extract paths of required submodules
118-
required_paths = {}
119-
for line in lines:
120-
if line.strip().startswith("path ="):
121-
path = line.split("=")[1].strip()
122-
for submodule, file_name in REQUIRED_SUBMODULES.items():
123-
if submodule in path:
124-
required_paths[path] = file_name
125-
return required_paths
126-
127-
128-
def check_and_update_submodules():
129-
def check_folder(folder: str, file: str) -> bool:
130-
return os.path.isdir(folder) and os.path.isfile(os.path.join(folder, file))
131-
132-
# Check if the directories exist for each required submodule
133-
missing_submodules = {}
134-
for path, file in get_required_submodule_paths().items():
135-
if not check_folder(path, file):
136-
missing_submodules[path] = file
137-
138-
# If any required submodule directories are missing, update them
139-
if missing_submodules:
140-
logger.warning("Some required submodules are missing. Updating submodules...")
141-
try:
142-
subprocess.check_call(["git", "submodule", "sync"])
143-
subprocess.check_call(["git", "submodule", "update", "--init"])
144-
except subprocess.CalledProcessError as e:
145-
logger.error(f"Error updating submodules: {e}")
146-
exit(1)
147-
148-
# After updating submodules, check again
149-
for path, file in missing_submodules.items():
150-
if not check_folder(path, file):
151-
logger.error(f"{file} not found in {path}.")
152-
logger.error("Please run `git submodule update --init`.")
153-
exit(1)
154-
logger.info("All required submodules are present.")
155-
15670

15771
class ShouldBuild:
15872
"""Indicates whether to build various components."""
@@ -825,39 +739,31 @@ def get_ext_modules() -> List[Extension]:
825739
return ext_modules
826740

827741

828-
def main():
829-
# Check submodules
830-
check_and_update_submodules()
831-
832-
setup(
833-
version=Version.string(),
834-
# TODO(dbort): Could use py_modules to restrict the set of modules we
835-
# package, and package_data to restrict the set up non-python files we
836-
# include. See also setuptools/discovery.py for custom finders.
837-
package_dir={
838-
"executorch/backends": "backends",
839-
"executorch/codegen": "codegen",
840-
# TODO(mnachin T180504136): Do not put examples/models
841-
# into core pip packages. Refactor out the necessary utils
842-
# or core models files into a separate package.
843-
"executorch/examples/models": "examples/models",
844-
"executorch/exir": "exir",
845-
"executorch/extension": "extension",
846-
"executorch/kernels/quantized": "kernels/quantized",
847-
"executorch/schema": "schema",
848-
"executorch/devtools": "devtools",
849-
"executorch/devtools/bundled_program": "devtools/bundled_program",
850-
"executorch/runtime": "runtime",
851-
"executorch/util": "util",
852-
},
853-
cmdclass={
854-
"build": CustomBuild,
855-
"build_ext": InstallerBuildExt,
856-
"build_py": CustomBuildPy,
857-
},
858-
ext_modules=get_ext_modules(),
859-
)
860-
861-
862-
if __name__ == "__main__":
863-
main()
742+
setup(
743+
version=Version.string(),
744+
# TODO(dbort): Could use py_modules to restrict the set of modules we
745+
# package, and package_data to restrict the set up non-python files we
746+
# include. See also setuptools/discovery.py for custom finders.
747+
package_dir={
748+
"executorch/backends": "backends",
749+
"executorch/codegen": "codegen",
750+
# TODO(mnachin T180504136): Do not put examples/models
751+
# into core pip packages. Refactor out the necessary utils
752+
# or core models files into a separate package.
753+
"executorch/examples/models": "examples/models",
754+
"executorch/exir": "exir",
755+
"executorch/extension": "extension",
756+
"executorch/kernels/quantized": "kernels/quantized",
757+
"executorch/schema": "schema",
758+
"executorch/devtools": "devtools",
759+
"executorch/devtools/bundled_program": "devtools/bundled_program",
760+
"executorch/runtime": "runtime",
761+
"executorch/util": "util",
762+
},
763+
cmdclass={
764+
"build": CustomBuild,
765+
"build_ext": InstallerBuildExt,
766+
"build_py": CustomBuildPy,
767+
},
768+
ext_modules=get_ext_modules(),
769+
)

0 commit comments

Comments
 (0)