Skip to content

Commit 117ae6e

Browse files
committed
Update on "[BE] Remind users to update submodule"
Summary: Fixing #7243 As titled. This PR adds 2 things: 1. `check_and_update_submodule()` check if required submodule folders exist and at least contains a CMakeLists.txt file. 2. Give useful error when the submodule is corrupted (most likely missing a file, caused by submodule out of sync). Test Plan: 1. Test if we can still install, if submodule is not updated ``` rm -rf third-party/prelude ./install_executorch.sh ``` See the following log ``` ... Processing /data/users/larryliu/executorch Preparing metadata (pyproject.toml): started Running command Preparing metadata (pyproject.toml) 2025-02-05 11:28:46,430 [ExecuTorch] WARNING: Some required submodules are missing. Updating submodules... Submodule path 'third-party/prelude': checked out '851d3f09c452937fc5adef27e2c50f7f304f1646' 2025-02-05 11:28:46,748 [ExecuTorch] INFO: All required submodules are present. 2025-02-05 11:28:47,018 [ExecuTorch] INFO: running dist_info ... ``` This proves that we can update submodule for the user. 2. Test if we can give useful error message, if we are missing a file. ``` rm third-party/gflags/src/gflags.cc ./install_executorch.sh ``` See the following error: Reviewers: Subscribers: Tasks: Tags: Differential Revision: [D69156975](https://our.internmc.facebook.com/intern/diff/D69156975) [ghstack-poisoned]
1 parent fa242fb commit 117ae6e

File tree

1 file changed

+54
-34
lines changed

1 file changed

+54
-34
lines changed

setup.py

Lines changed: 54 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@
5959
import subprocess
6060

6161
from distutils import log
62+
from distutils.errors import DistutilsExecError
6263
from distutils.sysconfig import get_python_lib
6364
from pathlib import Path
6465
from typing import List, Optional
@@ -89,56 +90,55 @@
8990
# too restrictive for users who modifies and tests the dependencies locally.
9091

9192
# keep sorted
92-
REQUIRED_SUBMODULES = [
93-
"ao",
94-
"cpuinfo",
95-
"eigen",
96-
"flatbuffers",
97-
"FP16",
98-
"FXdiv",
99-
"gflags",
100-
"prelude",
101-
"pthreadpool",
102-
"pybind11",
103-
"XNNPACK",
104-
]
93+
REQUIRED_SUBMODULES = {
94+
"ao": "LICENSE", # No CMakeLists.txt, choose a sort of stable file to check.
95+
"cpuinfo": "CMakeLists.txt",
96+
"eigen": "CMakeLists.txt",
97+
"flatbuffers": "CMakeLists.txt",
98+
"FP16": "CMakeLists.txt",
99+
"FXdiv": "CMakeLists.txt",
100+
"gflags": "CMakeLists.txt",
101+
"prelude": "BUCK",
102+
"pthreadpool": "CMakeLists.txt",
103+
"pybind11": "CMakeLists.txt",
104+
"XNNPACK": "CMakeLists.txt",
105+
}
105106

106107

107108
def get_required_submodule_paths():
108-
gitsubmodule_path = os.path.join(os.getcwd(), ".gitsubmodule")
109+
gitmodules_path = os.path.join(os.getcwd(), ".gitmodules")
109110

110-
if not os.path.isfile(gitsubmodule_path):
111-
print("Error: .gitsubmodule file not found.")
111+
if not os.path.isfile(gitmodules_path):
112+
logger.error(".gitmodules file not found.")
112113
exit(1)
113114

114-
with open(gitsubmodule_path, "r") as file:
115+
with open(gitmodules_path, "r") as file:
115116
lines = file.readlines()
116117

117118
# Extract paths of required submodules
118-
required_paths = []
119+
required_paths = {}
119120
for line in lines:
120121
if line.strip().startswith("path ="):
121122
path = line.split("=")[1].strip()
122-
if any(submodule in path for submodule in REQUIRED_SUBMODULES):
123-
required_paths.append(path)
123+
for submodule, file_name in REQUIRED_SUBMODULES.items():
124+
if submodule in path:
125+
required_paths[path] = file_name
124126
return required_paths
125127

126128

127129
def check_and_update_submodules():
128-
def check_folder(folder: str) -> bool:
129-
return os.path.isdir(folder) and os.path.isfile(
130-
os.path.join(folder, "CMakeLists.txt")
131-
)
130+
def check_folder(folder: str, file: str) -> bool:
131+
return os.path.isdir(folder) and os.path.isfile(os.path.join(folder, file))
132132

133133
# Check if the directories exist for each required submodule
134-
missing_submodules = []
135-
for path in get_required_submodule_paths():
136-
if not check_folder(path):
137-
missing_submodules.append(path)
134+
missing_submodules = {}
135+
for path, file in get_required_submodule_paths().items():
136+
if not check_folder(path, file):
137+
missing_submodules[path] = file
138138

139139
# If any required submodule directories are missing, update them
140140
if missing_submodules:
141-
logger.warn("Some required submodules are missing. Updating submodules...")
141+
logger.warning("Some required submodules are missing. Updating submodules...")
142142
try:
143143
subprocess.check_call(
144144
["git", "submodule", "update", "--init", "--recursive"]
@@ -148,12 +148,12 @@ def check_folder(folder: str) -> bool:
148148
exit(1)
149149

150150
# After updating submodules, check again
151-
for path in missing_submodules:
152-
if not check_folder(path):
153-
logger.error(f"Error: CMakeLists.txt not found in {path}.")
151+
for path, file in missing_submodules.items():
152+
if not check_folder(path, file):
153+
logger.error(f"{file} not found in {path}.")
154154
logger.error("Please run `git submodule update --init --recursive`.")
155155
exit(1)
156-
logger.info("All required submodules are present and contain CMakeLists.txt.")
156+
logger.info("All required submodules are present.")
157157

158158

159159
class ShouldBuild:
@@ -724,7 +724,27 @@ def run(self):
724724
# lists.
725725

726726
# Generate the build system files.
727-
self.spawn(["cmake", "-S", repo_root, "-B", cmake_cache_dir, *cmake_args])
727+
try:
728+
subprocess.run(
729+
["cmake", "-S", repo_root, "-B", cmake_cache_dir, *cmake_args],
730+
stdout=subprocess.PIPE,
731+
stderr=subprocess.PIPE,
732+
check=True,
733+
text=True,
734+
)
735+
except subprocess.CalledProcessError as e:
736+
error = str(e.stderr)
737+
# Our educated guesses from parsing the error message.
738+
# Missing source file, could be related to git submodules not synced or cmake cache is outdated
739+
additional_log = ""
740+
if "Cannot find source file" in error:
741+
additional_log = (
742+
"\033[31;1mEither CMake cache is outdated or git submodules are not synced.\n"
743+
"Please run the following before retry:\033[0m\n"
744+
" \033[32;1m./install_executorch.sh --clean\033[0m\n"
745+
" \033[32;1mgit submodule update --init --recursive\033[0m\n"
746+
)
747+
raise Exception(error + "\n" + additional_log) from e
728748

729749
# Build the system.
730750
self.spawn(["cmake", "--build", cmake_cache_dir, *build_args])

0 commit comments

Comments
 (0)