cmake: apply regex on sources to track dependencies

Author: madebr

cmake/FlatccGenerateSources.cmake

@@ -19,8 +19,7 @@ include(CMakeParseArguments)
 function(flatcc_generate_sources)
     # parse function arguments
     set(output_options SCHEMA COMMON COMMON_READER COMMON_BUILDER BUILDER READER VERIFIER JSON_PARSER JSON_PRINTER JSON)
-    # TODO: missing RECURSIVE option
-    set(NO_VAL_ARGS ALL ${output_options})
+    set(NO_VAL_ARGS ALL RECURSIVE ${output_options})
     set(SINGLE_VAL_ARGS OUTPUT_DIR OUTFILE PREFIX TARGET)
     set(MULTI_VAL_ARGS DEFINITION_FILES COMPILE_FLAGS PATHS)
 
@@ -36,9 +35,35 @@ function(flatcc_generate_sources)
 
     list(APPEND FLATCC_COMPILE_FLAGS -o "${FLATCC_OUTPUT_DIR}")
 
+    # Add current source directory for finding dependencies
+    set(absolute_flatcc_paths "${CMAKE_CURRENT_SOURCE_DIR}")
+    # Also add directory of definition files
+    foreach(definition_file FLATCC_DEFINITION_FILES)
+    endforeach()
+
+    set(ABSOLUTE_DEFINITION_FILES)
+    foreach(definition_file ${FLATCC_DEFINITION_FILES})
+        # TODO: file(REAL_PATH) if cmake_minimum_required_version >= 3.19
+        if(IS_ABSOLUTE "${definition_file}")
+            set(absolute_def_file "${definition_file}")
+        else()
+            set(absolute_def_file "${CMAKE_CURRENT_SOURCE_DIR}/${definition_file}")
+        endif()
+        list(APPEND ABSOLUTE_DEFINITION_FILES "${absolute_def_file}")
+
+        get_filename_component(absolute_def_directory "${absolute_def_file}" DIRECTORY)
+        list(APPEND absolute_flatcc_paths "${absolute_def_directory}")
+    endforeach()
+
     foreach(path ${FLATCC_PATHS})
+        # TODO: file(REAL_PATH) if cmake_minimum_required_version >= 3.19
+        if(NOT IS_ABSOLUTE "${path}")
+            set(path "${CMAKE_CURRENT_SOURCE_DIR}/${path}")
+        endif()
+        list(APPEND absolute_flatcc_paths "${path}")
         list(APPEND FLATCC_COMPILE_FLAGS -I "${path}")
     endforeach()
+    list(REMOVE_DUPLICATES absolute_flatcc_paths)
 
     # if no option is passed, the default READER is used
     set(default_option_reader ON)
@@ -86,10 +111,9 @@ function(flatcc_generate_sources)
     if (FLATCC_SCHEMA)
         list(APPEND FLATCC_COMPILE_FLAGS --schema)
     endif()
-    # TODO: missing recursive option
-    # if (FLATCC_RECURSIVE)
-    #     list(APPEND FLATCC_COMPILE_FLAGS --recursive)
-    # endif()
+    if (FLATCC_RECURSIVE)
+        list(APPEND FLATCC_COMPILE_FLAGS --recursive)
+    endif()
 
     # handle 'all', 'common', 'json' last as they encompass other options
     if (FLATCC_COMMON)
@@ -109,8 +133,6 @@ function(flatcc_generate_sources)
         set(FLATCC_COMMON_READER ON)
         set(FLATCC_BUILDER ON)
         set(FLATCC_VERIFIER ON)
-        # TODO: missing recursive option
-        # set(FLATCC_RECURSIVE ON)
     endif()
 
     # Calculate suffixes of output files.
@@ -135,24 +157,58 @@ function(flatcc_generate_sources)
         list(APPEND GENERATED_FILE_SUFFIXES .bfbs)
     endif()
 
-    set(ABSOLUTE_DEFINITION_FILES)
-
-    foreach(definition_file ${FLATCC_DEFINITION_FILES})
-        # TODO: file(REAL_PATH) if cmake_minimum_Version >= 3.19
-        if(IS_ABSOLUTE "${definition_file}")
-            set(absolute_def_file "${definition_file}")
-        else()
-            set(absolute_def_file "${CMAKE_CURRENT_SOURCE_DIR}/${definition_file}")
+    # grep each definition file recursively for includes, convert them to absolute paths and add them to a list
+    set(ABSOLUTE_DEFINITIONS_DEPENDENCIES)
+    set(absolute_definition_files_todo ${ABSOLUTE_DEFINITION_FILES})
+    while(absolute_definition_files_todo)
+        list(GET absolute_definition_files_todo 0 current_deffile)
+        # TODO: use if(absolute_definition_files_todo IN_LIST ABSOLUTE_DEFINITIONS_DEPENDENCIES) if cmake_minimum_required_version >= 3.3
+        list(FIND ABSOLUTE_DEFINITIONS_DEPENDENCIES "${current_deffile}" todo_index)
+        if(todo_index LESS 0)
+            list(APPEND ABSOLUTE_DEFINITIONS_DEPENDENCIES "${current_deffile}")
+            file(READ "${current_deffile}" contents_deffile)
+            set(include_regex  "(^|\n)include[ \t]+\"([^\"]+)\"")
+            string(REGEX MATCHALL "${include_regex}" includes_contents_match "${contents_deffile}")
+            foreach(include_match ${includes_contents_match})
+                string(REGEX MATCH "${include_regex}" _m "${include_match}")
+                set(include_def_file "${CMAKE_MATCH_2}")
+                if(IS_ABSOLUTE "${include_def_file}")
+                    set(abs_include_def_file "${include_def_file}")
+                else()
+                    set(abs_include_def_file)
+                    foreach(absolute_flatcc_path ${absolute_flatcc_paths})
+                        if(NOT abs_include_def_file)
+                            if(EXISTS "${absolute_flatcc_path}/${include_def_file}")
+                                set(abs_include_def_file "${absolute_flatcc_path}/${include_def_file}")
+                            endif()
+                        endif()
+                    endforeach()
+                    if(NOT abs_include_def_file)
+                        message(WARNING "${current_deffile} includes ${include_def_file}, but cannot be found in search path")
+                    endif()
+                endif()
+                if(abs_include_def_file)
+                    list(APPEND absolute_definition_files_todo "${abs_include_def_file}")
+                endif()
+            endforeach()
         endif()
-        list(APPEND ABSOLUTE_DEFINITION_FILES "${absolute_def_file}")
-    endforeach()
+        list(REMOVE_AT absolute_definition_files_todo 0)
+    endwhile()
+
+    list(REMOVE_DUPLICATES ABSOLUTE_DEFINITIONS_DEPENDENCIES)
+    list(REMOVE_DUPLICATES ABSOLUTE_DEFINITION_FILES)
 
     set(OUTPUT_FILES)
     if(FLATCC_OUTFILE)
         list(APPEND FLATCC_COMPILE_FLAGS "--outfile=${FLATCC_OUTPUT_DIR}/${FLATCC_OUTFILE}")
         list(APPEND OUTPUT_FILES "${FLATCC_OUTPUT_DIR}/${FLATCC_OUTFILE}")
     else()
-        foreach(definition_file ${FLATCC_DEFINITION_FILES})
+        if(FLATCC_RECURSIVE)
+            set(definition_files ${ABSOLUTE_DEFINITIONS_DEPENDENCIES})
+        else()
+            set(definition_files ${ABSOLUTE_DEFINITION_FILES})
+        endif()
+        foreach(definition_file ${definition_files})
             get_filename_component(def_name_we "${definition_file}" NAME_WLE)
             foreach(suffix ${GENERATED_FILE_SUFFIXES})
                 list(APPEND OUTPUT_FILES "${FLATCC_OUTPUT_DIR}/${def_name_we}${suffix}")
@@ -167,16 +223,20 @@ function(flatcc_generate_sources)
         endif()
     endif()
 
-    message(VERBOSE "Flatcc output directory: ${FLATCC_OUTPUT_DIR}")
-    message(VERBOSE "Flatcc output files: ${OUTPUT_FILES}")
-    message(VERBOSE "Flatcc execute: flatcc;${FLATCC_COMPILE_FLAGS};${ABSOLUTE_DEFINITION_FILES}")
+    message(VERBOSE "---- flatcc start ----")
+    message(VERBOSE "output directory: ${FLATCC_OUTPUT_DIR}")
+    message(VERBOSE "output files: ${OUTPUT_FILES}")
+    message(VERBOSE "execute: flatcc;${FLATCC_COMPILE_FLAGS};${ABSOLUTE_DEFINITION_FILES}")
+    message(VERBOSE "dependencies: ${ABSOLUTE_DEFINITIONS_DEPENDENCIES}")
+    message(VERBOSE "----- flatcc end -----")
+
     add_custom_command(OUTPUT ${OUTPUT_FILES}
         COMMAND "${CMAKE_COMMAND}" -E make_directory "${FLATCC_OUTPUT_DIR}"
         COMMAND flatcc::cli ${FLATCC_COMPILE_FLAGS} ${ABSOLUTE_DEFINITION_FILES}
-        DEPENDS flatcc::cli ${FLATCC_DEFINITION_FILES}
+        DEPENDS flatcc::cli ${ABSOLUTE_DEFINITIONS_DEPENDENCIES}
     )
     if(FLATCC_TARGET)
-        # FIXME: possible improvement if cmake_minimum_required >= 3.1:
+        # TODO: possible improvement if cmake_minimum_required_version >= 3.1:
         #        use add_library(... INTERFACE) +  add_custom_command + target_sources(INTERFACE) + target_link_libraries()
         add_custom_target("${FLATCC_TARGET}" DEPENDS ${OUTPUT_FILES})
     endif()

test/flatc_compat/CMakeLists.txt

@@ -3,9 +3,8 @@ set(FBS_DIR "${PROJECT_SOURCE_DIR}/test/monster_test")
 
 include_directories("${GEN_DIR}")
 
-# TODO: use recursive
 flatcc_generate_sources(ALL OUTPUT_DIR "${GEN_DIR}"
-    DEFINITION_FILES "${FBS_DIR}/monster_test.fbs" "${FBS_DIR}/include_test1.fbs" "${FBS_DIR}/include_test2.fbs")
+    DEFINITION_FILES "${FBS_DIR}/monster_test.fbs")
 
 add_executable(flatc_compat flatc_compat.c "${GEN_DIR}/monster_test_reader.h" "${GEN_DIR}/monster_test_verifier.h")
 target_link_libraries(flatc_compat flatcc::runtime)

test/json_test/CMakeLists.txt

@@ -6,12 +6,8 @@ set(DATA_SRC "${PROJECT_SOURCE_DIR}/test/flatc_compat")
 
 include_directories("${GEN_DIR}")
 
-# FIXME: possible improvement if cmake_minimum_required >= 3.1:
-#        use add_library(... INTERFACE) +  add_custom_command + target_sources(INTERFACE) + target_link_libraries()
-
-
 flatcc_generate_sources(ALL VERIFIER JSON OUTPUT_DIR "${GEN_DIR}" TARGET gen_json_test
-    DEFINITION_FILES "${FBS_DIR}/monster_test.fbs" "${FBS_DIR}/include_test1.fbs" "${FBS_DIR}/include_test2.fbs")
+    DEFINITION_FILES "${FBS_DIR}/monster_test.fbs")
 
 add_custom_command (TARGET gen_json_test POST_BUILD
     COMMAND cmake -E copy "${DATA_SRC}/monsterdata_test.golden" "${DATA_DST}"

test/load_test/CMakeLists.txt

@@ -3,9 +3,8 @@ set(FBS_DIR "${PROJECT_SOURCE_DIR}/test/monster_test")
 
 include_directories("${GEN_DIR}")
 
-# TODO: use recursive
 flatcc_generate_sources(ALL OUTPUT_DIR "${GEN_DIR}"
-    DEFINITION_FILES "${FBS_DIR}/monster_test.fbs" "${FBS_DIR}/include_test1.fbs" "${FBS_DIR}/include_test2.fbs")
+    DEFINITION_FILES "${FBS_DIR}/monster_test.fbs")
 
 add_executable(load_test load_test.c "${GEN_DIR}/monster_test_builder.h")
 target_link_libraries(load_test flatcc::runtime)

test/monster_test/CMakeLists.txt

@@ -3,9 +3,8 @@ set(FBS_DIR "${PROJECT_SOURCE_DIR}/test/monster_test")
 
 include_directories("${GEN_DIR}")
 
-# TODO: use recursive
 flatcc_generate_sources(ALL OUTPUT_DIR "${GEN_DIR}"
-    DEFINITION_FILES "${FBS_DIR}/monster_test.fbs" "${FBS_DIR}/include_test1.fbs" "${FBS_DIR}/include_test2.fbs")
+    DEFINITION_FILES "${FBS_DIR}/monster_test.fbs")
 
 add_executable(monster_test monster_test.c "${GEN_DIR}/monster_test_builder.h" "${GEN_DIR}/monster_test_verifier.h")
 target_link_libraries(monster_test flatcc::runtime)

test/monster_test_concat/CMakeLists.txt

@@ -6,7 +6,7 @@ include_directories("${GEN_DIR}")
 flatcc_generate_sources(ALL
     OUTPUT_DIR "${GEN_DIR}"
     OUTFILE "monster_test.h"
-    DEFINITION_FILES "${FBS_DIR}/monster_test.fbs" "${FBS_DIR}/include_test1.fbs" "${FBS_DIR}/include_test2.fbs")
+    DEFINITION_FILES "${FBS_DIR}/monster_test.fbs")
 
 add_executable(monster_test_concat monster_test_concat.c "${GEN_DIR}/monster_test.h")
 target_link_libraries(monster_test_concat flatcc::runtime)

test/monster_test_prefix/CMakeLists.txt

@@ -6,7 +6,7 @@ include_directories("${GEN_DIR}")
 flatcc_generate_sources(ALL PREFIX zzz_
     OUTPUT_DIR "${GEN_DIR}"
     OUTFILE "zzz_monster_test.h"
-    DEFINITION_FILES "${FBS_DIR}/monster_test.fbs" "${FBS_DIR}/include_test1.fbs" "${FBS_DIR}/include_test2.fbs")
+    DEFINITION_FILES "${FBS_DIR}/monster_test.fbs")
 
 add_executable(monster_test_prefix monster_test_prefix.c "${GEN_DIR}/zzz_monster_test.h")
 target_link_libraries(monster_test_prefix flatcc::runtime)

test/reflection_test/CMakeLists.txt

@@ -2,7 +2,7 @@ set(GEN_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated")
 set(FBS_DIR "${PROJECT_SOURCE_DIR}/test/monster_test")
 
 flatcc_generate_sources(SCHEMA OUTPUT_DIR "${GEN_DIR}"
-    DEFINITION_FILES "${FBS_DIR}/monster_test.fbs" "${FBS_DIR}/include_test1.fbs" "${FBS_DIR}/include_test2.fbs")
+    DEFINITION_FILES "${FBS_DIR}/monster_test.fbs")
 
 add_executable(reflection_test reflection_test.c "${GEN_DIR}/monster_test.bfbs")
 target_link_libraries(reflection_test flatcc::runtime)