diff --git a/.ci/scripts/unittest-linux.sh b/.ci/scripts/unittest-linux.sh index d0f107ed338..d9ce2fa50d7 100755 --- a/.ci/scripts/unittest-linux.sh +++ b/.ci/scripts/unittest-linux.sh @@ -30,6 +30,7 @@ elif [[ "$BUILD_TOOL" == "buck2" ]]; then # Removing this breaks sccache in the Buck build, apparently # because TMPDIR gets messed up? Please feel free to fix this and # speed up this CI job! + CMAKE_ARGS="-DEXECUTORCH_FETCH_GFLAGS_FOR_BUCK=ON" \ PYTHON_EXECUTABLE=python \ .ci/scripts/setup-linux.sh "$@" diff --git a/.gitignore b/.gitignore index e4b6af2bbad..c20f930d3cf 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,9 @@ ethos-u-scratch/ executorch.egg-info pip-out/ +# Third-party libraries, symlink'ed to FetchContent source directories +third-party/gflags + # Any exported models and profiling outputs *.bin *.model diff --git a/.gitmodules b/.gitmodules index 702bf091f21..8c6cdd9f8d3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -40,9 +40,6 @@ [submodule "third-party/flatcc"] path = third-party/flatcc url = https://github.com/dvidelabs/flatcc.git -[submodule "third-party/gflags"] - path = third-party/gflags - url = https://github.com/gflags/gflags.git [submodule "third-party/googletest"] path = third-party/googletest url = https://github.com/google/googletest.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 0540c2da039..737fe1b5be5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -52,8 +52,10 @@ project(executorch) include(${PROJECT_SOURCE_DIR}/tools/cmake/common/preset.cmake) include(${PROJECT_SOURCE_DIR}/tools/cmake/Utils.cmake) +include(${PROJECT_SOURCE_DIR}/tools/cmake/FetchContent.cmake) include(CMakeDependentOption) include(ExternalProject) +include(FetchContent) if(NOT CMAKE_CXX_STANDARD) set(CMAKE_CXX_STANDARD 17) @@ -456,14 +458,6 @@ endif() add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/configurations) -# -# gflags: Commandline flag host library. -# - -if(EXECUTORCH_BUILD_GFLAGS) - add_subdirectory(third-party/gflags) -endif() - # Install `executorch` library as well as `executorch-config.cmake` under # ${CMAKE_INSTALL_PREFIX}/ install( @@ -703,6 +697,7 @@ endif() if(EXECUTORCH_BUILD_EXECUTOR_RUNNER) # Baseline libraries that executor_runner will link against. + FetchContent_gflags() set(_executor_runner_libs executorch gflags) if(EXECUTORCH_BUILD_KERNELS_OPTIMIZED) diff --git a/backends/apple/coreml/scripts/build_tests.sh b/backends/apple/coreml/scripts/build_tests.sh index 890385d5e24..190adf1f65a 100755 --- a/backends/apple/coreml/scripts/build_tests.sh +++ b/backends/apple/coreml/scripts/build_tests.sh @@ -33,8 +33,7 @@ cmake "$EXECUTORCH_ROOT_PATH" -B"$CMAKE_EXECUTORCH_BUILD_DIR_PATH" \ -DPLATFORM=MAC_UNIVERSAL \ -DDEPLOYMENT_TARGET=13.0 \ -DEXECUTORCH_BUILD_EXECUTOR_RUNNER=OFF \ --DEXECUTORCH_BUILD_XNNPACK=OFF \ --DEXECUTORCH_BUILD_GFLAGS=OFF +-DEXECUTORCH_BUILD_XNNPACK=OFF cmake --build "$CMAKE_EXECUTORCH_BUILD_DIR_PATH" -j9 -t executorch diff --git a/backends/cadence/build_cadence_fusionG3.sh b/backends/cadence/build_cadence_fusionG3.sh index 3ced26f8577..20e694206db 100644 --- a/backends/cadence/build_cadence_fusionG3.sh +++ b/backends/cadence/build_cadence_fusionG3.sh @@ -50,7 +50,6 @@ if $STEPWISE_BUILD; then -DEXECUTORCH_BUILD_KERNELS_CUSTOM=OFF \ -DPYTHON_EXECUTABLE=python3 \ -DEXECUTORCH_FUSION_G3_OPT=ON \ - -DEXECUTORCH_BUILD_GFLAGS=ON \ -DHAVE_FNMATCH_H=OFF \ -Bcmake-out/backends/cadence \ backends/cadence diff --git a/backends/cadence/build_cadence_hifi4.sh b/backends/cadence/build_cadence_hifi4.sh index 70f026fe0b2..fab8febcef5 100644 --- a/backends/cadence/build_cadence_hifi4.sh +++ b/backends/cadence/build_cadence_hifi4.sh @@ -49,7 +49,6 @@ if $STEPWISE_BUILD; then -DEXECUTORCH_BUILD_KERNELS_CUSTOM=OFF \ -DPYTHON_EXECUTABLE=python3 \ -DEXECUTORCH_NNLIB_OPT=ON \ - -DEXECUTORCH_BUILD_GFLAGS=ON \ -DHAVE_FNMATCH_H=OFF \ -Bcmake-out/backends/cadence \ backends/cadence diff --git a/examples/models/llama/CMakeLists.txt b/examples/models/llama/CMakeLists.txt index 8c27de20845..2994923cf7c 100644 --- a/examples/models/llama/CMakeLists.txt +++ b/examples/models/llama/CMakeLists.txt @@ -47,6 +47,7 @@ set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../..) set(TORCH_ROOT ${EXECUTORCH_ROOT}/third-party/pytorch) include(${EXECUTORCH_ROOT}/tools/cmake/Utils.cmake) +include(${EXECUTORCH_ROOT}/tools/cmake/FetchContent.cmake) if(NOT PYTHON_EXECUTABLE) resolve_python_executable() @@ -68,10 +69,8 @@ set(_common_compile_options -Wno-deprecated-declarations -fPIC) # Let files say "include ". set(_common_include_directories ${EXECUTORCH_ROOT}/..) -# For some reason android build is not able to find where gflags is and hence -# cannot find corresponding .cmake file -set(gflags_DIR ${CMAKE_CURRENT_BINARY_DIR}/../../../third-party/gflags) -find_package(gflags REQUIRED) +# Fetch gflags +FetchContent_gflags() # # llama_main: test binary to run llama, with tokenizer and sampler integrated diff --git a/examples/models/llava/CMakeLists.txt b/examples/models/llava/CMakeLists.txt index 64be6111674..9e964f770d9 100644 --- a/examples/models/llava/CMakeLists.txt +++ b/examples/models/llava/CMakeLists.txt @@ -45,6 +45,7 @@ endif() set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../..) include(${EXECUTORCH_ROOT}/tools/cmake/Utils.cmake) +include(${EXECUTORCH_ROOT}/tools/cmake/FetchContent.cmake) if(NOT PYTHON_EXECUTABLE) resolve_python_executable() @@ -66,10 +67,8 @@ set(_common_compile_options -Wno-deprecated-declarations -fPIC) # Let files say "include ". set(_common_include_directories ${EXECUTORCH_ROOT}/..) -# For some reason android build is not able to find where gflags is and hence -# cannot find corresponding .cmake file -set(gflags_DIR ${CMAKE_CURRENT_BINARY_DIR}/../../../third-party/gflags) -find_package(gflags REQUIRED) +# Fetch gflags +FetchContent_gflags() # # llava_main: test binary to run llava, with tokenizer and sampler integrated diff --git a/examples/portable/custom_ops/CMakeLists.txt b/examples/portable/custom_ops/CMakeLists.txt index 5a9a9a11fe6..61f157a298e 100644 --- a/examples/portable/custom_ops/CMakeLists.txt +++ b/examples/portable/custom_ops/CMakeLists.txt @@ -29,6 +29,7 @@ endif() include(${EXECUTORCH_ROOT}/tools/cmake/Utils.cmake) include(${EXECUTORCH_ROOT}/tools/cmake/Codegen.cmake) +include(${EXECUTORCH_ROOT}/tools/cmake/FetchContent.cmake) if(NOT PYTHON_EXECUTABLE) resolve_python_executable() @@ -40,9 +41,7 @@ set(_common_compile_options -Wno-deprecated-declarations -fPIC) set(_common_include_directories ${EXECUTORCH_ROOT}/..) find_package(executorch CONFIG REQUIRED) -find_package( - gflags REQUIRED PATHS ${CMAKE_CURRENT_BINARY_DIR}/../../../third-party -) +FetchContent_gflags() target_include_directories(executorch INTERFACE ${_common_include_directories}) diff --git a/examples/qualcomm/CMakeLists.txt b/examples/qualcomm/CMakeLists.txt index 4b0e6b2d3a2..14faa175d6d 100644 --- a/examples/qualcomm/CMakeLists.txt +++ b/examples/qualcomm/CMakeLists.txt @@ -30,7 +30,6 @@ endif() # etdump, bundled_program. find_package(executorch CONFIG REQUIRED) target_compile_options(executorch INTERFACE -DET_EVENT_TRACER_ENABLED) -find_package(gflags REQUIRED) set(_common_compile_options -Wno-deprecated-declarations -fPIC) diff --git a/examples/selective_build/CMakeLists.txt b/examples/selective_build/CMakeLists.txt index a37b4362d78..8002e68adf0 100644 --- a/examples/selective_build/CMakeLists.txt +++ b/examples/selective_build/CMakeLists.txt @@ -23,6 +23,7 @@ set(TORCH_ROOT ${EXECUTORCH_ROOT}/third-party/pytorch) include(${EXECUTORCH_ROOT}/tools/cmake/Utils.cmake) include(${EXECUTORCH_ROOT}/tools/cmake/Codegen.cmake) +include(${EXECUTORCH_ROOT}/tools/cmake/FetchContent.cmake) if(NOT PYTHON_EXECUTABLE) resolve_python_executable() @@ -39,9 +40,7 @@ set(_common_compile_options -Wno-deprecated-declarations -fPIC) set(_common_include_directories ${EXECUTORCH_ROOT}/..) find_package(executorch CONFIG REQUIRED) -find_package( - gflags REQUIRED PATHS ${CMAKE_CURRENT_BINARY_DIR}/../../third-party -) +FetchContent_gflags() target_include_directories(executorch_core INTERFACE ${_common_include_directories}) diff --git a/extension/training/CMakeLists.txt b/extension/training/CMakeLists.txt index 55537359cfe..5edbda1a0b8 100644 --- a/extension/training/CMakeLists.txt +++ b/extension/training/CMakeLists.txt @@ -31,6 +31,7 @@ target_link_libraries(extension_training executorch_core kernels_util_all_deps list(TRANSFORM _train_xor__srcs PREPEND "${EXECUTORCH_ROOT}/") add_executable(train_xor ${_train_xor__srcs}) + target_include_directories( train_xor PUBLIC ${_common_include_directories} ) diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt index d7c9c5774fb..5cc4cf6893f 100644 --- a/third-party/CMakeLists.txt +++ b/third-party/CMakeLists.txt @@ -14,6 +14,15 @@ if(BUILD_TESTING) add_subdirectory(googletest) endif() +if(EXECUTORCH_FETCH_GFLAGS_FOR_BUCK) + FetchContent_gflags() + # Create a symlink from the gflags source directory to third-party/gflags + message(STATUS "Creating a symlink from ${gflags_SOURCE_DIR} to third-party/gflags") + execute_process( + COMMAND ${CMAKE_COMMAND} -E create_symlink "${gflags_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/gflags" + ) +endif() + # MARK: - flatbuffers if(WIN32) diff --git a/third-party/gflags b/third-party/gflags deleted file mode 160000 index a738fdf9338..00000000000 --- a/third-party/gflags +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a738fdf9338412f83ab3f26f31ac11ed3f3ec4bd diff --git a/third-party/gflags.bzl b/third-party/gflags.bzl index 205bb3bd1d0..53d66da82e7 100644 --- a/third-party/gflags.bzl +++ b/third-party/gflags.bzl @@ -2,10 +2,13 @@ load(":gflags_defs.bzl", "gflags_library", "gflags_sources") def define_gflags(): - (exported_headers, headers, srcs) = gflags_sources(namespace = [ - "gflags", - "google", - ]) + if native.read_config("et_oss", "enable_gflags", "true") == "true": + (exported_headers, headers, srcs) = gflags_sources(namespace = [ + "gflags", + "google", + ]) + else: + (exported_headers, headers, srcs) = ([], [], []) gflags_library( name = "gflags", diff --git a/tools/cmake/FetchContent.cmake b/tools/cmake/FetchContent.cmake new file mode 100644 index 00000000000..7396163a049 --- /dev/null +++ b/tools/cmake/FetchContent.cmake @@ -0,0 +1,54 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# FetchContent gflags and make available at CMake configure time. +# Make the content available at configure time so that BUCK can use it. +include(FetchContent) + +# Some third-party libraries need to be materialized in the source tree for +# buck. Unfortunately, FetchContent bakes in generator information in the +# fetched content. Thus, if the generator is changed, it causes build failures. +# So, if the generator has changed, then nuke the content. +function(_invalidate_cache_if_generator_mismatch DIR) + set(_generator_stamp_file "${DIR}/.executorch_cmake_generator_stamp") + set(_current_generator "${CMAKE_GENERATOR}") + + if(EXISTS "${_generator_stamp_file}") + file(READ "${_generator_stamp_file}" _previous_generator) + string(STRIP "${_previous_generator}" _previous_generator) + if(NOT _previous_generator STREQUAL _current_generator) + file(REMOVE_RECURSE ${DIR}) + endif() + endif() + +endfunction() + +# Fetch gflags, and make a symlink to third-party/gflags in the source tree. +# Doing this to satisfy BUCK query for gflags. Also try to invalidate the cmake +# cache if the generator has changed. Notice that symlink won't be created again +# if it's already there. +function(FetchContent_gflags) + # set(_symlink_target ${CMAKE_CURRENT_LIST_DIR}/../../third-party/gflags) + # if(IS_DIRECTORY ${_symlink_target}) + # _invalidate_cache_if_generator_mismatch(${_symlink_target}) + # endif() + + FetchContent_Declare( + gflags + GIT_REPOSITORY https://github.com/gflags/gflags.git + GIT_TAG v2.2.2 + ) + set(GFLAGS_INTTYPES_FORMAT C99) + FetchContent_MakeAvailable(gflags) + + # if(NOT IS_DIRECTORY ${_symlink_target}) + # message(STATUS "Creating a symlink from ${gflags_SOURCE_DIR} to third-party/gflags") + # execute_process( + # COMMAND ${CMAKE_COMMAND} -E create_symlink "${gflags_SOURCE_DIR}" "${_symlink_target}" + # ) + # file(WRITE "${_symlink_target}/.executorch_cmake_generator_stamp" "${CMAKE_GENERATOR}") + # endif() +endfunction() diff --git a/tools/cmake/extract_sources.py b/tools/cmake/extract_sources.py index 5af0904fdfd..7f7888d7f9b 100755 --- a/tools/cmake/extract_sources.py +++ b/tools/cmake/extract_sources.py @@ -129,7 +129,7 @@ def get_sources( # If we hit here, setup.py:check_submodule() should have already run # but it could be that the submodules are not synced or there's local changes. try: - sources: set[str] = set(runner.run(["cquery", query] + buck_args)) + sources: set[str] = set(runner.run(["cquery", "--config", "et_oss.enable_gflags=false", query] + buck_args)) except RuntimeError as e: logger.error( f"\033[31;1mFailed to query buck for sources. Failed command:\n\n" diff --git a/tools/cmake/preset/default.cmake b/tools/cmake/preset/default.cmake index 2d0c6302117..a1e5fc71249 100644 --- a/tools/cmake/preset/default.cmake +++ b/tools/cmake/preset/default.cmake @@ -210,9 +210,9 @@ define_overridable_option( BOOL OFF ) define_overridable_option( - EXECUTORCH_BUILD_GFLAGS - "Build the gflags library." - BOOL ON + EXECUTORCH_FETCH_GFLAGS_FOR_BUCK + "Fetch gflags for BUCK build/test" + BOOL OFF ) define_overridable_option( EXECUTORCH_COREML_BUILD_EXECUTOR_RUNNER diff --git a/tools/cmake/preset/llm.cmake b/tools/cmake/preset/llm.cmake index da1364eb2ad..9bd97d9ea7e 100644 --- a/tools/cmake/preset/llm.cmake +++ b/tools/cmake/preset/llm.cmake @@ -16,6 +16,7 @@ set_overridable_option(EXECUTORCH_BUILD_KERNELS_CUSTOM ON) set_overridable_option(EXECUTORCH_BUILD_KERNELS_OPTIMIZED ON) set_overridable_option(EXECUTORCH_BUILD_KERNELS_QUANTIZED ON) set_overridable_option(EXECUTORCH_BUILD_XNNPACK ON) +set_overridable_option(EXECUTORCH_FETCH_GFLAGS_FOR_BUCK ON) if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") set_overridable_option(EXECUTORCH_BUILD_COREML ON)