[RFC] Fetch gflags dynamically

.gitignore

@@ -21,6 +21,9 @@ ethos-u-scratch/
 executorch.egg-info
 pip-out/
 
+# Dynamically fetched third-party libraries
+third-party/gflags
+
 # Any exported models and profiling outputs
 *.bin
 *.model

.gitmodules

@@ -40,9 +40,6 @@
 [submodule "third-party/flatcc"]
 	path = third-party/flatcc
 	url = https://github.com/dvidelabs/flatcc.git
-[submodule "third-party/gflags"]
-	path = third-party/gflags
-	url = https://github.com/gflags/gflags.git
 [submodule "third-party/googletest"]
 	path = third-party/googletest
 	url = https://github.com/google/googletest.git

CMakeLists.txt

@@ -52,8 +52,10 @@ project(executorch)
 
 include(${PROJECT_SOURCE_DIR}/tools/cmake/common/preset.cmake)
 include(${PROJECT_SOURCE_DIR}/tools/cmake/Utils.cmake)
+include(${PROJECT_SOURCE_DIR}/tools/cmake/EXECUTORCH_FetchContent_Destination.cmake)
 include(CMakeDependentOption)
 include(ExternalProject)
+include(FetchContent)
 
 if(NOT CMAKE_CXX_STANDARD)
   set(CMAKE_CXX_STANDARD 17)
@@ -456,14 +458,6 @@ endif()
 
 add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/configurations)
 
-#
-# gflags: Commandline flag host library.
-#
-
-if(EXECUTORCH_BUILD_GFLAGS)
-  add_subdirectory(third-party/gflags)
-endif()
-
 # Install `executorch` library as well as `executorch-config.cmake` under
 # ${CMAKE_INSTALL_PREFIX}/
 install(

backends/apple/coreml/scripts/build_tests.sh

@@ -33,8 +33,7 @@ cmake "$EXECUTORCH_ROOT_PATH" -B"$CMAKE_EXECUTORCH_BUILD_DIR_PATH" \
 -DPLATFORM=MAC_UNIVERSAL \
 -DDEPLOYMENT_TARGET=13.0 \
 -DEXECUTORCH_BUILD_EXECUTOR_RUNNER=OFF \
--DEXECUTORCH_BUILD_XNNPACK=OFF \
--DEXECUTORCH_BUILD_GFLAGS=OFF
+-DEXECUTORCH_BUILD_XNNPACK=OFF
 
 cmake --build "$CMAKE_EXECUTORCH_BUILD_DIR_PATH"  -j9 -t executorch
 

backends/cadence/build_cadence_fusionG3.sh

@@ -50,7 +50,6 @@ if $STEPWISE_BUILD; then
         -DEXECUTORCH_BUILD_KERNELS_CUSTOM=OFF \
         -DPYTHON_EXECUTABLE=python3 \
         -DEXECUTORCH_FUSION_G3_OPT=ON \
-        -DEXECUTORCH_BUILD_GFLAGS=ON \
         -DHAVE_FNMATCH_H=OFF \
         -Bcmake-out/backends/cadence \
         backends/cadence

backends/cadence/build_cadence_hifi4.sh

@@ -49,7 +49,6 @@ if $STEPWISE_BUILD; then
         -DEXECUTORCH_BUILD_KERNELS_CUSTOM=OFF \
         -DPYTHON_EXECUTABLE=python3 \
         -DEXECUTORCH_NNLIB_OPT=ON \
-        -DEXECUTORCH_BUILD_GFLAGS=ON \
         -DHAVE_FNMATCH_H=OFF \
         -Bcmake-out/backends/cadence \
         backends/cadence

examples/models/llama/CMakeLists.txt

@@ -70,7 +70,7 @@ set(_common_include_directories ${EXECUTORCH_ROOT}/..)
 
 # For some reason android build is not able to find where gflags is and hence
 # cannot find corresponding .cmake file
-set(gflags_DIR ${CMAKE_CURRENT_BINARY_DIR}/../../../third-party/gflags)
+set(gflags_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../../third-party/gflags/gflags-build)
 find_package(gflags REQUIRED)
 
 #

examples/models/llava/CMakeLists.txt

@@ -68,7 +68,7 @@ set(_common_include_directories ${EXECUTORCH_ROOT}/..)
 
 # For some reason android build is not able to find where gflags is and hence
 # cannot find corresponding .cmake file
-set(gflags_DIR ${CMAKE_CURRENT_BINARY_DIR}/../../../third-party/gflags)
+set(gflags_DIR ${CMAKE_CURRENT_BINARY_DIR}/../../../third-party/gflags/gflags-build)
 find_package(gflags REQUIRED)
 
 #

examples/portable/custom_ops/CMakeLists.txt

@@ -40,9 +40,8 @@ set(_common_compile_options -Wno-deprecated-declarations -fPIC)
 set(_common_include_directories ${EXECUTORCH_ROOT}/..)
 
 find_package(executorch CONFIG REQUIRED)
-find_package(
-  gflags REQUIRED PATHS ${CMAKE_CURRENT_BINARY_DIR}/../../../third-party
-)
+set(gflags_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../../third-party/gflags/gflags-build)
+find_package(gflags REQUIRED)
 
 target_include_directories(executorch INTERFACE ${_common_include_directories})
 

examples/qualcomm/CMakeLists.txt

@@ -30,6 +30,8 @@ endif()
 # etdump, bundled_program.
 find_package(executorch CONFIG REQUIRED)
 target_compile_options(executorch INTERFACE -DET_EVENT_TRACER_ENABLED)
+
+set(gflags_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../third-party/gflags/gflags-build)
 find_package(gflags REQUIRED)
 
 set(_common_compile_options -Wno-deprecated-declarations -fPIC)

examples/selective_build/CMakeLists.txt

@@ -39,9 +39,8 @@ set(_common_compile_options -Wno-deprecated-declarations -fPIC)
 set(_common_include_directories ${EXECUTORCH_ROOT}/..)
 
 find_package(executorch CONFIG REQUIRED)
-find_package(
-  gflags REQUIRED PATHS ${CMAKE_CURRENT_BINARY_DIR}/../../third-party
-)
+set(gflags_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../third-party/gflags/gflags-build)
+find_package(gflags REQUIRED)
 
 target_include_directories(executorch_core INTERFACE ${_common_include_directories})
 

third-party/CMakeLists.txt

@@ -14,6 +14,15 @@ if(BUILD_TESTING)
   add_subdirectory(googletest)
 endif()
 
+EXECUTORCH_FetchContent_Destination("${CMAKE_CURRENT_SOURCE_DIR}/gflags")
+FetchContent_Declare(
+  gflags
+  GIT_REPOSITORY https://github.com/gflags/gflags.git
+  GIT_TAG v2.2.2
+)
+set(GFLAGS_INTTYPES_FORMAT C99)
+FetchContent_MakeAvailable(gflags)
+
 # MARK: - flatbuffers
 
 if(WIN32)

third-party/gflags_defs.bzl

@@ -1,4 +1,9 @@
 # ------------------------------------------------------------------------------
+
+# gflags is materialized during build time by cmake. See third-party/CMakeLists.txt.
+def _path(relative_path):
+    return "gflags/gflags-src/{}".format(relative_path)
+
 # Add native rules to configure source files
 # Not tested for building on windows platforms
 def gflags_sources(namespace = ["google", "gflags"]):
@@ -7,7 +12,7 @@ def gflags_sources(namespace = ["google", "gflags"]):
     # @lint-ignore BUCKLINT: native and fb_native are explicitly forbidden in fbcode.
     native.genrule(
         name = "gflags_declare_h",
-        srcs = ["gflags/src/gflags_declare.h.in"],
+        srcs = [_path("src/gflags_declare.h.in")],
         out = "gflags/gflags_declare.h",
         cmd = (common_preamble + "awk '{ " +
                "gsub(/@GFLAGS_NAMESPACE@/, \"" + namespace[0] + "\"); " +
@@ -22,7 +27,7 @@ def gflags_sources(namespace = ["google", "gflags"]):
         # @lint-ignore BUCKLINT: native and fb_native are explicitly forbidden in fbcode.
         native.genrule(
             name = gflags_ns_h_file.replace(".", "_"),
-            srcs = ["gflags/src/gflags_ns.h.in"],
+            srcs = [_path("src/gflags_ns.h.in")],
             out = "gflags/" + gflags_ns_h_file,
             cmd = (common_preamble + "awk '{ " +
                    "gsub(/@ns@/, \"" + ns + "\"); " +
@@ -34,7 +39,7 @@ def gflags_sources(namespace = ["google", "gflags"]):
     # @lint-ignore BUCKLINT: native and fb_native are explicitly forbidden in fbcode.
     native.genrule(
         name = "gflags_h",
-        srcs = ["gflags/src/gflags.h.in"],
+        srcs = [_path("src/gflags.h.in")],
         out = "gflags/gflags.h",
         cmd = (common_preamble + "awk '{ " +
                "gsub(/@GFLAGS_ATTRIBUTE_UNUSED@/, \"\"); " +
@@ -45,15 +50,15 @@ def gflags_sources(namespace = ["google", "gflags"]):
     # @lint-ignore BUCKLINT: native and fb_native are explicitly forbidden in fbcode.
     native.genrule(
         name = "gflags_completions_h",
-        srcs = ["gflags/src/gflags_completions.h.in"],
+        srcs = [_path("src/gflags_completions.h.in")],
         out = "gflags/gflags_completions.h",
         cmd = common_preamble + "awk '{ gsub(/@GFLAGS_NAMESPACE@/, \"" + namespace[0] + "\"); print; }' $SRCS > $OUT",
     )
     headers = {
-        "config.h": "gflags/src/config.h",
-        "mutex.h": "gflags/src/mutex.h",
-        "util.h": "gflags/src/util.h",
-        "windows_port.h": "gflags/src/windows_port.h",
+        "config.h": _path("src/config.h"),
+        "mutex.h": _path("src/mutex.h"),
+        "util.h": _path("src/util.h"),
+        "windows_port.h": _path("src/windows_port.h"),
     }
     exported_headers = {
         "gflags/gflags.h": ":gflags_h",
@@ -62,9 +67,9 @@ def gflags_sources(namespace = ["google", "gflags"]):
     }
     exported_headers.update({"gflags/" + hdr: ":" + hdr.replace(".", "_") for hdr in gflags_ns_h_files})
     srcs = [
-        "gflags/src/gflags.cc",
-        "gflags/src/gflags_completions.cc",
-        "gflags/src/gflags_reporting.cc",
+        _path("src/gflags.cc"),
+        _path("src/gflags_completions.cc"),
+        _path("src/gflags_reporting.cc"),
     ]
     return [exported_headers, headers, srcs]
 

tools/cmake/EXECUTORCH_FetchContent_Destination.cmake

@@ -0,0 +1,30 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# Some third-party libraries need to be materialized in the source tree for
+# buck. Unfortunately, FetchContent bakes in generator information in the
+# fetched content. Thus, if the generator is changed, it causes build failures.
+# So, if the generator has changed, then nuke the content.
+function(_refresh_cache_if_necessary DIR)
+  set(_generator_stamp_file "${DIR}/.executorch_cmake_generator_stamp")
+  set(_current_generator "${CMAKE_GENERATOR}")
+
+  if(EXISTS "${_generator_stamp_file}")
+    file(READ "${_generator_stamp_file}" _previous_generator)
+    string(STRIP "${_previous_generator}" _previous_generator)
+    if(NOT _previous_generator STREQUAL _current_generator)
+      file(REMOVE_RECURSE ${DIR})
+    endif()
+  endif()
+
+  file(WRITE "${_generator_stamp_file}" "${_current_generator}")
+endfunction()
+
+# Declare the content to be fetched in a specific location.
+macro(EXECUTORCH_FetchContent_Destination DIR)
+  _refresh_cache_if_necessary(${DIR})
+  set(FETCHCONTENT_BASE_DIR ${DIR})
+endmacro()

tools/cmake/preset/default.cmake

@@ -209,11 +209,6 @@ define_overridable_option(
   "Build the Cortex-M backend"
   BOOL OFF
 )
-define_overridable_option(
-  EXECUTORCH_BUILD_GFLAGS
-  "Build the gflags library."
-  BOOL ON
-)
 define_overridable_option(
   EXECUTORCH_COREML_BUILD_EXECUTOR_RUNNER
   "Build CoreML executor runner."