[RFC] Fetch gflags dynamically

.ci/scripts/unittest-linux.sh

@@ -30,6 +30,7 @@ elif [[ "$BUILD_TOOL" == "buck2" ]]; then
     # Removing this breaks sccache in the Buck build, apparently
     # because TMPDIR gets messed up? Please feel free to fix this and
     # speed up this CI job!
+    CMAKE_ARGS="-DEXECUTORCH_FETCH_GFLAGS_FOR_BUCK=ON" \
     PYTHON_EXECUTABLE=python \
     .ci/scripts/setup-linux.sh "$@"
 

.gitignore

@@ -21,6 +21,9 @@ ethos-u-scratch/
 executorch.egg-info
 pip-out/
 
+# Third-party libraries, symlink'ed to FetchContent source directories
+third-party/gflags
+
 # Any exported models and profiling outputs
 *.bin
 *.model

.gitmodules

@@ -40,9 +40,6 @@
 [submodule "third-party/flatcc"]
 	path = third-party/flatcc
 	url = https://github.com/dvidelabs/flatcc.git
-[submodule "third-party/gflags"]
-	path = third-party/gflags
-	url = https://github.com/gflags/gflags.git
 [submodule "third-party/googletest"]
 	path = third-party/googletest
 	url = https://github.com/google/googletest.git

CMakeLists.txt

@@ -52,8 +52,10 @@ project(executorch)
 
 include(${PROJECT_SOURCE_DIR}/tools/cmake/common/preset.cmake)
 include(${PROJECT_SOURCE_DIR}/tools/cmake/Utils.cmake)
+include(${PROJECT_SOURCE_DIR}/tools/cmake/FetchContent.cmake)
 include(CMakeDependentOption)
 include(ExternalProject)
+include(FetchContent)
 
 if(NOT CMAKE_CXX_STANDARD)
   set(CMAKE_CXX_STANDARD 17)
@@ -456,14 +458,6 @@ endif()
 
 add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/configurations)
 
-#
-# gflags: Commandline flag host library.
-#
-
-if(EXECUTORCH_BUILD_GFLAGS)
-  add_subdirectory(third-party/gflags)
-endif()
-
 # Install `executorch` library as well as `executorch-config.cmake` under
 # ${CMAKE_INSTALL_PREFIX}/
 install(
@@ -703,6 +697,7 @@ endif()
 
 if(EXECUTORCH_BUILD_EXECUTOR_RUNNER)
   # Baseline libraries that executor_runner will link against.
+  FetchContent_gflags()
   set(_executor_runner_libs executorch gflags)
 
   if(EXECUTORCH_BUILD_KERNELS_OPTIMIZED)

backends/apple/coreml/scripts/build_tests.sh

@@ -33,8 +33,7 @@ cmake "$EXECUTORCH_ROOT_PATH" -B"$CMAKE_EXECUTORCH_BUILD_DIR_PATH" \
 -DPLATFORM=MAC_UNIVERSAL \
 -DDEPLOYMENT_TARGET=13.0 \
 -DEXECUTORCH_BUILD_EXECUTOR_RUNNER=OFF \
--DEXECUTORCH_BUILD_XNNPACK=OFF \
--DEXECUTORCH_BUILD_GFLAGS=OFF
+-DEXECUTORCH_BUILD_XNNPACK=OFF
 
 cmake --build "$CMAKE_EXECUTORCH_BUILD_DIR_PATH"  -j9 -t executorch
 

backends/cadence/build_cadence_fusionG3.sh

@@ -50,7 +50,6 @@ if $STEPWISE_BUILD; then
         -DEXECUTORCH_BUILD_KERNELS_CUSTOM=OFF \
         -DPYTHON_EXECUTABLE=python3 \
         -DEXECUTORCH_FUSION_G3_OPT=ON \
-        -DEXECUTORCH_BUILD_GFLAGS=ON \
         -DHAVE_FNMATCH_H=OFF \
         -Bcmake-out/backends/cadence \
         backends/cadence

backends/cadence/build_cadence_hifi4.sh

@@ -49,7 +49,6 @@ if $STEPWISE_BUILD; then
         -DEXECUTORCH_BUILD_KERNELS_CUSTOM=OFF \
         -DPYTHON_EXECUTABLE=python3 \
         -DEXECUTORCH_NNLIB_OPT=ON \
-        -DEXECUTORCH_BUILD_GFLAGS=ON \
         -DHAVE_FNMATCH_H=OFF \
         -Bcmake-out/backends/cadence \
         backends/cadence

examples/models/llama/CMakeLists.txt

@@ -47,6 +47,7 @@ set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../..)
 set(TORCH_ROOT ${EXECUTORCH_ROOT}/third-party/pytorch)
 
 include(${EXECUTORCH_ROOT}/tools/cmake/Utils.cmake)
+include(${EXECUTORCH_ROOT}/tools/cmake/FetchContent.cmake)
 
 if(NOT PYTHON_EXECUTABLE)
   resolve_python_executable()
@@ -68,10 +69,8 @@ set(_common_compile_options -Wno-deprecated-declarations -fPIC)
 # Let files say "include <executorch/path/to/header.h>".
 set(_common_include_directories ${EXECUTORCH_ROOT}/..)
 
-# For some reason android build is not able to find where gflags is and hence
-# cannot find corresponding .cmake file
-set(gflags_DIR ${CMAKE_CURRENT_BINARY_DIR}/../../../third-party/gflags)
-find_package(gflags REQUIRED)
+# Fetch gflags
+FetchContent_gflags()
 
 #
 # llama_main: test binary to run llama, with tokenizer and sampler integrated

examples/models/llava/CMakeLists.txt

@@ -45,6 +45,7 @@ endif()
 set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../..)
 
 include(${EXECUTORCH_ROOT}/tools/cmake/Utils.cmake)
+include(${EXECUTORCH_ROOT}/tools/cmake/FetchContent.cmake)
 
 if(NOT PYTHON_EXECUTABLE)
   resolve_python_executable()
@@ -66,10 +67,8 @@ set(_common_compile_options -Wno-deprecated-declarations -fPIC)
 # Let files say "include <executorch/path/to/header.h>".
 set(_common_include_directories ${EXECUTORCH_ROOT}/..)
 
-# For some reason android build is not able to find where gflags is and hence
-# cannot find corresponding .cmake file
-set(gflags_DIR ${CMAKE_CURRENT_BINARY_DIR}/../../../third-party/gflags)
-find_package(gflags REQUIRED)
+# Fetch gflags
+FetchContent_gflags()
 
 #
 # llava_main: test binary to run llava, with tokenizer and sampler integrated

examples/portable/custom_ops/CMakeLists.txt

@@ -29,6 +29,7 @@ endif()
 
 include(${EXECUTORCH_ROOT}/tools/cmake/Utils.cmake)
 include(${EXECUTORCH_ROOT}/tools/cmake/Codegen.cmake)
+include(${EXECUTORCH_ROOT}/tools/cmake/FetchContent.cmake)
 
 if(NOT PYTHON_EXECUTABLE)
   resolve_python_executable()
@@ -40,9 +41,7 @@ set(_common_compile_options -Wno-deprecated-declarations -fPIC)
 set(_common_include_directories ${EXECUTORCH_ROOT}/..)
 
 find_package(executorch CONFIG REQUIRED)
-find_package(
-  gflags REQUIRED PATHS ${CMAKE_CURRENT_BINARY_DIR}/../../../third-party
-)
+FetchContent_gflags()
 
 target_include_directories(executorch INTERFACE ${_common_include_directories})
 

examples/qualcomm/CMakeLists.txt

@@ -30,7 +30,6 @@ endif()
 # etdump, bundled_program.
 find_package(executorch CONFIG REQUIRED)
 target_compile_options(executorch INTERFACE -DET_EVENT_TRACER_ENABLED)
-find_package(gflags REQUIRED)
 
 set(_common_compile_options -Wno-deprecated-declarations -fPIC)
 

examples/selective_build/CMakeLists.txt

@@ -23,6 +23,7 @@ set(TORCH_ROOT ${EXECUTORCH_ROOT}/third-party/pytorch)
 
 include(${EXECUTORCH_ROOT}/tools/cmake/Utils.cmake)
 include(${EXECUTORCH_ROOT}/tools/cmake/Codegen.cmake)
+include(${EXECUTORCH_ROOT}/tools/cmake/FetchContent.cmake)
 
 if(NOT PYTHON_EXECUTABLE)
   resolve_python_executable()
@@ -39,9 +40,7 @@ set(_common_compile_options -Wno-deprecated-declarations -fPIC)
 set(_common_include_directories ${EXECUTORCH_ROOT}/..)
 
 find_package(executorch CONFIG REQUIRED)
-find_package(
-  gflags REQUIRED PATHS ${CMAKE_CURRENT_BINARY_DIR}/../../third-party
-)
+FetchContent_gflags()
 
 target_include_directories(executorch_core INTERFACE ${_common_include_directories})
 

extension/training/CMakeLists.txt

@@ -31,6 +31,7 @@ target_link_libraries(extension_training executorch_core kernels_util_all_deps
 
 list(TRANSFORM _train_xor__srcs PREPEND "${EXECUTORCH_ROOT}/")
 add_executable(train_xor ${_train_xor__srcs})
+
 target_include_directories(
   train_xor PUBLIC ${_common_include_directories}
 )

third-party/CMakeLists.txt

@@ -14,6 +14,15 @@ if(BUILD_TESTING)
   add_subdirectory(googletest)
 endif()
 
+if(EXECUTORCH_FETCH_GFLAGS_FOR_BUCK)
+  FetchContent_gflags()
+  # Create a symlink from the gflags source directory to third-party/gflags
+  message(STATUS "Creating a symlink from ${gflags_SOURCE_DIR} to third-party/gflags")
+  execute_process(
+    COMMAND ${CMAKE_COMMAND} -E create_symlink "${gflags_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/gflags"
+  )
+endif()
+
 # MARK: - flatbuffers
 
 if(WIN32)

third-party/gflags.bzl

@@ -2,10 +2,13 @@
 load(":gflags_defs.bzl", "gflags_library", "gflags_sources")
 
 def define_gflags():
-    (exported_headers, headers, srcs) = gflags_sources(namespace = [
-        "gflags",
-        "google",
-    ])
+    if native.read_config("et_oss", "enable_gflags", "true") == "true":
+        (exported_headers, headers, srcs) = gflags_sources(namespace = [
+            "gflags",
+            "google",
+        ])
+    else:
+        (exported_headers, headers, srcs) = ([], [], [])
 
     gflags_library(
         name = "gflags",

tools/cmake/FetchContent.cmake

@@ -0,0 +1,54 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# FetchContent gflags and make available at CMake configure time.
+# Make the content available at configure time so that BUCK can use it.
+include(FetchContent)
+
+# Some third-party libraries need to be materialized in the source tree for
+# buck. Unfortunately, FetchContent bakes in generator information in the
+# fetched content. Thus, if the generator is changed, it causes build failures.
+# So, if the generator has changed, then nuke the content.
+function(_invalidate_cache_if_generator_mismatch DIR)
+  set(_generator_stamp_file "${DIR}/.executorch_cmake_generator_stamp")
+  set(_current_generator "${CMAKE_GENERATOR}")
+
+  if(EXISTS "${_generator_stamp_file}")
+    file(READ "${_generator_stamp_file}" _previous_generator)
+    string(STRIP "${_previous_generator}" _previous_generator)
+    if(NOT _previous_generator STREQUAL _current_generator)
+      file(REMOVE_RECURSE ${DIR})
+    endif()
+  endif()
+
+endfunction()
+
+# Fetch gflags, and make a symlink to third-party/gflags in the source tree.
+# Doing this to satisfy BUCK query for gflags. Also try to invalidate the cmake
+# cache if the generator has changed. Notice that symlink won't be created again
+# if it's already there.
+function(FetchContent_gflags)
+  # set(_symlink_target ${CMAKE_CURRENT_LIST_DIR}/../../third-party/gflags)
+  # if(IS_DIRECTORY ${_symlink_target})
+  #   _invalidate_cache_if_generator_mismatch(${_symlink_target})
+  # endif()
+
+  FetchContent_Declare(
+    gflags
+    GIT_REPOSITORY https://github.com/gflags/gflags.git
+    GIT_TAG v2.2.2
+  )
+  set(GFLAGS_INTTYPES_FORMAT C99)
+  FetchContent_MakeAvailable(gflags)
+
+  # if(NOT IS_DIRECTORY ${_symlink_target})
+  #   message(STATUS "Creating a symlink from ${gflags_SOURCE_DIR} to third-party/gflags")
+  #   execute_process(
+  #     COMMAND ${CMAKE_COMMAND} -E create_symlink "${gflags_SOURCE_DIR}" "${_symlink_target}"
+  #   )
+  #   file(WRITE "${_symlink_target}/.executorch_cmake_generator_stamp" "${CMAKE_GENERATOR}")
+  # endif()
+endfunction()

tools/cmake/extract_sources.py

@@ -129,7 +129,7 @@ def get_sources(
         # If we hit here, setup.py:check_submodule() should have already run
         # but it could be that the submodules are not synced or there's local changes.
         try:
-            sources: set[str] = set(runner.run(["cquery", query] + buck_args))
+            sources: set[str] = set(runner.run(["cquery", "--config", "et_oss.enable_gflags=false", query] + buck_args))
         except RuntimeError as e:
             logger.error(
                 f"\033[31;1mFailed to query buck for sources. Failed command:\n\n"

tools/cmake/preset/default.cmake

@@ -210,9 +210,9 @@ define_overridable_option(
   BOOL OFF
 )
 define_overridable_option(
-  EXECUTORCH_BUILD_GFLAGS
-  "Build the gflags library."
-  BOOL ON
+  EXECUTORCH_FETCH_GFLAGS_FOR_BUCK
+  "Fetch gflags for BUCK build/test"
+  BOOL OFF
 )
 define_overridable_option(
   EXECUTORCH_COREML_BUILD_EXECUTOR_RUNNER

tools/cmake/preset/llm.cmake

@@ -16,6 +16,7 @@ set_overridable_option(EXECUTORCH_BUILD_KERNELS_CUSTOM ON)
 set_overridable_option(EXECUTORCH_BUILD_KERNELS_OPTIMIZED ON)
 set_overridable_option(EXECUTORCH_BUILD_KERNELS_QUANTIZED ON)
 set_overridable_option(EXECUTORCH_BUILD_XNNPACK ON)
+set_overridable_option(EXECUTORCH_FETCH_GFLAGS_FOR_BUCK ON)
 
 if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
   set_overridable_option(EXECUTORCH_BUILD_COREML ON)