[BE] Remind users to update submodule

Summary: As titled. This adds messages when we suspect the user forgot
to update git submodule or cleanup the CMake cache.

Test Plan: See the following error messages

Reviewers:

Subscribers:

Tasks:

Tags:

ghstack-source-id: 14d8e65
Pull Request resolved: #8203

Author: larryliu0820

build/extract_sources.py

@@ -7,6 +7,7 @@
 
 import argparse
 import copy
+import logging
 import os
 import re
 
@@ -18,7 +19,7 @@
 try:
     import tomllib  # Standard in 3.11 and later
 except ModuleNotFoundError:
-    import tomli as tomllib
+    import tomli as tomllib  # type: ignore[no-redef]
 
 """Extracts source lists from the buck2 build system and writes them to a file.
 
@@ -66,6 +67,12 @@
     ]
 """
 
+# Set up logging
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s [ExecuTorch] %(levelname)s: %(message)s"
+)
+logger = logging.getLogger()
+
 
 class Target:
     """Parsed [targets.*] entry from the TOML file.
@@ -118,7 +125,22 @@ def get_sources(
         )
 
         # Get the complete list of source files that this target depends on.
-        sources: set[str] = set(runner.run(["cquery", query] + buck_args))
+        # If user doesn't setup their git submodules correctly, this will fail.
+        # If we hit here, setup.py:check_submodule() should have already run
+        # but it could be that the submodules are not synced or there's local changes.
+        try:
+            sources: set[str] = set(runner.run(["cquery", query] + buck_args))
+        except RuntimeError as e:
+            logger.error(
+                f"\033[31;1mFailed to query buck for sources. Failed command:\n\n"
+                f"   buck2 cquery {query} {' '.join(buck_args)}\n\n"
+                "This is likely due "
+                "to missing git submodules or outdated CMake cache. "
+                "Please run the following before retry:\033[0m\n\n"
+                "    \033[32;1m./install_executorch.sh --clean\033[0m\n"
+                "    \033[32;1mgit submodule update --init --recursive\033[0m\n"
+            )
+            raise e
 
         # Keep entries that match all of the filters.
         filters = [re.compile(p) for p in self._config.get("filters", [])]

setup.py

@@ -55,6 +55,8 @@
 # Import this before distutils so that setuptools can intercept the distuils
 # imports.
 import setuptools  # noqa: F401 # usort: skip
+import logging
+import subprocess
 
 from distutils import log
 from distutils.sysconfig import get_python_lib
@@ -66,9 +68,93 @@
 from setuptools.command.build_ext import build_ext
 from setuptools.command.build_py import build_py
 
+# Set up logging
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s [ExecuTorch] %(levelname)s: %(message)s"
+)
+logger = logging.getLogger()
+
 # For information on setuptools Command subclassing see
 # https://setuptools.pypa.io/en/latest/userguide/extension.html
 
+################################################################################
+# Git submodules
+################################################################################
+# The following submodules are required to be able to build ExecuTorch. If any of
+# these folders are missing or missing CMakeLists.txt, we will run
+# `git submodule update` to try to fix it. If the command fails, we will raise an
+# error.
+# An alternative to this would be to run `git submodule status` and run
+# `git submodule update` if there's any local changes. However this is a bit
+# too restrictive for users who modifies and tests the dependencies locally.
+
+# keep sorted
+REQUIRED_SUBMODULES = [
+    "ao",
+    "cpuinfo",
+    "eigen",
+    "flatbuffers",
+    "FP16",
+    "FXdiv",
+    "gflags",
+    "prelude",
+    "pthreadpool",
+    "pybind11",
+    "XNNPACK",
+]
+
+
+def get_required_submodule_paths():
+    gitsubmodule_path = os.path.join(os.getcwd(), ".gitsubmodule")
+
+    if not os.path.isfile(gitsubmodule_path):
+        print("Error: .gitsubmodule file not found.")
+        exit(1)
+
+    with open(gitsubmodule_path, "r") as file:
+        lines = file.readlines()
+
+    # Extract paths of required submodules
+    required_paths = []
+    for line in lines:
+        if line.strip().startswith("path ="):
+            path = line.split("=")[1].strip()
+            if any(submodule in path for submodule in REQUIRED_SUBMODULES):
+                required_paths.append(path)
+    return required_paths
+
+
+def check_and_update_submodules():
+    def check_folder(folder: str) -> bool:
+        return os.path.isdir(folder) and os.path.isfile(
+            os.path.join(folder, "CMakeLists.txt")
+        )
+
+    # Check if the directories exist for each required submodule
+    missing_submodules = []
+    for path in get_required_submodule_paths():
+        if not check_folder(path):
+            missing_submodules.append(path)
+
+    # If any required submodule directories are missing, update them
+    if missing_submodules:
+        logger.warn("Some required submodules are missing. Updating submodules...")
+        try:
+            subprocess.check_call(
+                ["git", "submodule", "update", "--init", "--recursive"]
+            )
+        except subprocess.CalledProcessError as e:
+            logger.error(f"Error updating submodules: {e}")
+            exit(1)
+
+        # After updating submodules, check again
+        for path in missing_submodules:
+            if not check_folder(path):
+                logger.error(f"Error: CMakeLists.txt not found in {path}.")
+                logger.error("Please run `git submodule update --init --recursive`.")
+                exit(1)
+    logger.info("All required submodules are present and contain CMakeLists.txt.")
+
 
 class ShouldBuild:
     """Indicates whether to build various components."""
@@ -720,31 +806,39 @@ def get_ext_modules() -> List[Extension]:
     return ext_modules
 
 
-setup(
-    version=Version.string(),
-    # TODO(dbort): Could use py_modules to restrict the set of modules we
-    # package, and package_data to restrict the set up non-python files we
-    # include. See also setuptools/discovery.py for custom finders.
-    package_dir={
-        "executorch/backends": "backends",
-        "executorch/codegen": "codegen",
-        # TODO(mnachin T180504136): Do not put examples/models
-        # into core pip packages. Refactor out the necessary utils
-        # or core models files into a separate package.
-        "executorch/examples/models": "examples/models",
-        "executorch/exir": "exir",
-        "executorch/extension": "extension",
-        "executorch/kernels/quantized": "kernels/quantized",
-        "executorch/schema": "schema",
-        "executorch/devtools": "devtools",
-        "executorch/devtools/bundled_program": "devtools/bundled_program",
-        "executorch/runtime": "runtime",
-        "executorch/util": "util",
-    },
-    cmdclass={
-        "build": CustomBuild,
-        "build_ext": InstallerBuildExt,
-        "build_py": CustomBuildPy,
-    },
-    ext_modules=get_ext_modules(),
-)
+def main():
+    # Check submodules
+    check_and_update_submodules()
+
+    setup(
+        version=Version.string(),
+        # TODO(dbort): Could use py_modules to restrict the set of modules we
+        # package, and package_data to restrict the set up non-python files we
+        # include. See also setuptools/discovery.py for custom finders.
+        package_dir={
+            "executorch/backends": "backends",
+            "executorch/codegen": "codegen",
+            # TODO(mnachin T180504136): Do not put examples/models
+            # into core pip packages. Refactor out the necessary utils
+            # or core models files into a separate package.
+            "executorch/examples/models": "examples/models",
+            "executorch/exir": "exir",
+            "executorch/extension": "extension",
+            "executorch/kernels/quantized": "kernels/quantized",
+            "executorch/schema": "schema",
+            "executorch/devtools": "devtools",
+            "executorch/devtools/bundled_program": "devtools/bundled_program",
+            "executorch/runtime": "runtime",
+            "executorch/util": "util",
+        },
+        cmdclass={
+            "build": CustomBuild,
+            "build_ext": InstallerBuildExt,
+            "build_py": CustomBuildPy,
+        },
+        ext_modules=get_ext_modules(),
+    )
+
+
+if __name__ == "__main__":
+    main()