Set up CMake configure dependencies on all JSON files used in the configuration (#409)

* Set up CMake configure dependencies on all JSON files used in the configuration

* Remove unneeded cmake-build-dir option, we already have output

* Oops use namespace

* Fix comment

* Don't explode if given paths outside the root dir

Author: Jamie Smith

tools/cmake/mbed_generate_configuration.cmake

@@ -27,57 +27,43 @@ else()
 endif()
 set(MBED_INTERNAL_LAST_MBED_TARGET "${MBED_TARGET}" CACHE INTERNAL "Previous mbed target this dir was configured with" FORCE)
 
-# Convert all relative paths to absolute paths, rooted at CMAKE_SOURCE_DIR.
-# This makes sure that they are interpreted the same way everywhere.
-get_filename_component(MBED_APP_JSON_PATH "${MBED_APP_JSON_PATH}" ABSOLUTE BASE_DIR ${CMAKE_SOURCE_DIR})
-get_filename_component(CUSTOM_TARGETS_JSON_PATH "${CUSTOM_TARGETS_JSON_PATH}" ABSOLUTE BASE_DIR ${CMAKE_SOURCE_DIR})
-
-# Make it so that if mbed_app.json or custom_targets.json are modified, CMake is rerun.
-set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${MBED_APP_JSON_PATH})
-if(EXISTS "${CUSTOM_TARGETS_JSON_PATH}" AND (NOT IS_DIRECTORY "${CUSTOM_TARGETS_JSON_PATH}"))
-    set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${CUSTOM_TARGETS_JSON_PATH})
-endif()
-
-# Check if mbed_app.json was modified
-# Note: if the path is an empty string, get_filename_component(ABSOLUTE) will convert it to a directory,
-# so we have to verify that the path we have is a file, not a dir.
-if(EXISTS "${MBED_APP_JSON_PATH}" AND (NOT IS_DIRECTORY "${MBED_APP_JSON_PATH}"))
-    file(TIMESTAMP "${MBED_APP_JSON_PATH}" MBED_APP_JSON_TIMESTAMP "%s" UTC)
-else()
-    set(MBED_APP_JSON_TIMESTAMP "<none>")
-endif()
-
-if(NOT MBED_NEED_TO_RECONFIGURE)
-    if(NOT "${MBED_INTERNAL_LAST_MBED_APP_JSON_TIMESTAMP}" STREQUAL "${MBED_APP_JSON_TIMESTAMP}")
-        message(STATUS "Mbed: mbed_app.json modified, regenerating configs...")
+# Check if the include file is missing (e.g. because a previous attempt to generate it failed)
+if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/mbed_config.cmake)
+    if(NOT MBED_NEED_TO_RECONFIGURE)
+        message(STATUS "Mbed: mbed_config.cmake not found, regenerating configs...")
         set(MBED_NEED_TO_RECONFIGURE TRUE)
     endif()
-endif()
-set(MBED_INTERNAL_LAST_MBED_APP_JSON_TIMESTAMP "${MBED_APP_JSON_TIMESTAMP}" CACHE INTERNAL "Previous UTC modification timestamp for mbed_app.json" FORCE)
-
-# Check if custom_targets.json was modified
-if(EXISTS "${CUSTOM_TARGETS_JSON_PATH}" AND (NOT IS_DIRECTORY "${CUSTOM_TARGETS_JSON_PATH}"))
-    file(TIMESTAMP "${CUSTOM_TARGETS_JSON_PATH}" CUSTOM_TARGETS_JSON_TIMESTAMP "%s" UTC)
 else()
-    set(CUSTOM_TARGETS_JSON_TIMESTAMP "<none>")
+    # Include the old version of mbed_config.cmake to get the MBED_CONFIG_JSON_SOURCE_FILES variable used below
+    include(${CMAKE_CURRENT_BINARY_DIR}/mbed_config.cmake)
 endif()
 
+# Check timestamps on all JSON files used to generate the Mbed configuration
 if(NOT MBED_NEED_TO_RECONFIGURE)
-    if(NOT "${MBED_INTERNAL_LAST_CUSTOM_TARGETS_JSON_TIMESTAMP}" STREQUAL "${CUSTOM_TARGETS_JSON_TIMESTAMP}")
-        message(STATUS "Mbed: custom_targets.json modified, regenerating configs...")
-        set(MBED_NEED_TO_RECONFIGURE TRUE)
-    endif()
+    file(TIMESTAMP ${CMAKE_CURRENT_BINARY_DIR}/mbed_config.cmake MBED_CONFIG_CMAKE_TIMESTAMP "%s" UTC)
+
+    foreach(CONFIG_JSON ${MBED_CONFIG_JSON_SOURCE_FILES})
+        get_filename_component(CONFIG_JSON_ABSPATH ${CONFIG_JSON} ABSOLUTE)
+
+        if(NOT EXISTS ${CONFIG_JSON_ABSPATH})
+            message(STATUS "Mbed: ${CONFIG_JSON} deleted or renamed, regenerating configs...")
+            set(MBED_NEED_TO_RECONFIGURE TRUE)
+            break()
+        endif()
+
+        file(TIMESTAMP ${CONFIG_JSON_ABSPATH} CONFIG_JSON_TIMESTAMP "%s" UTC)
+        if(${CONFIG_JSON_TIMESTAMP} GREATER ${MBED_CONFIG_CMAKE_TIMESTAMP})
+            message(STATUS "Mbed: ${CONFIG_JSON} modified, regenerating configs...")
+            set(MBED_NEED_TO_RECONFIGURE TRUE)
+            break()
+        endif()
+    endforeach()
 endif()
 
-set(MBED_INTERNAL_LAST_CUSTOM_TARGETS_JSON_TIMESTAMP "${CUSTOM_TARGETS_JSON_TIMESTAMP}" CACHE INTERNAL "Previous UTC modification timestamp for custom_targets.json" FORCE)
-
-# Check if the include file is missing (e.g. because a previous attempt to generate it failed)
-if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/mbed_config.cmake)
-    if(NOT MBED_NEED_TO_RECONFIGURE)
-        message(STATUS "Mbed: mbed_config.cmake not found, regenerating configs...")
-        set(MBED_NEED_TO_RECONFIGURE TRUE)
-    endif()
-endif()
+# Convert all relative paths to absolute paths, rooted at CMAKE_SOURCE_DIR.
+# This makes sure that they are interpreted the same way everywhere.
+get_filename_component(MBED_APP_JSON_PATH "${MBED_APP_JSON_PATH}" ABSOLUTE BASE_DIR ${CMAKE_SOURCE_DIR})
+get_filename_component(CUSTOM_TARGETS_JSON_PATH "${CUSTOM_TARGETS_JSON_PATH}" ABSOLUTE BASE_DIR ${CMAKE_SOURCE_DIR})
 
 if(MBED_NEED_TO_RECONFIGURE)
     # Generate mbed_config.cmake for this target
@@ -98,7 +84,7 @@ if(MBED_NEED_TO_RECONFIGURE)
 
     set(MBEDTOOLS_CONFIGURE_COMMAND ${Python3_EXECUTABLE}
         -m mbed_tools.cli.main
-        -v # without -v, warnings (e.g. "you have tried to override a nonexistent parameter") do not get printed
+        -v -v # without at least -v, warnings (e.g. "you have tried to override a nonexistent parameter") do not get printed
         configure
         -t GCC_ARM # GCC_ARM is currently the only supported toolchain
         -m "${MBED_TARGET}"
@@ -126,4 +112,7 @@ if(MBED_NEED_TO_RECONFIGURE)
 endif()
 
 # Include the generated config file
-include(${CMAKE_CURRENT_BINARY_DIR}/mbed_config.cmake)
+include(${CMAKE_CURRENT_BINARY_DIR}/mbed_config.cmake)
+
+# Make it so that if any config JSON files are modified, CMake is rerun.
+set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${MBED_CONFIG_JSON_SOURCE_FILES})

tools/python/mbed_tools/build/_internal/cmake_file.py

@@ -30,7 +30,11 @@ def render_mbed_config_cmake_template(config: Config, toolchain_name: str, targe
     env.filters["to_hex"] = to_hex
     template = env.get_template(TEMPLATE_NAME)
     config["supported_c_libs"] = [x for x in config["supported_c_libs"][toolchain_name.lower()]]
-    context = {"target_name": target_name, "toolchain_name": toolchain_name, **config}
+
+    context = {"target_name": target_name,
+               "toolchain_name": toolchain_name,
+               "json_sources": config.json_sources,
+               **config}
     return template.render(context)
 
 

tools/python/mbed_tools/build/_internal/config/assemble_build_config.py

@@ -9,12 +9,15 @@
 from pathlib import Path
 from typing import Iterable, List, Optional, Set
 
+from setuptools.build_meta import build_editable
+
+from mbed_tools.project import MbedProgram
 from mbed_tools.build._internal.config.config import Config
 from mbed_tools.build._internal.config import source
 from mbed_tools.build._internal.find_files import LabelFilter, RequiresFilter, filter_files, find_files
 
 
-def assemble_config(target_attributes: dict, search_paths: Iterable[Path], mbed_app_file: Optional[Path]) -> Config:
+def assemble_config(target_attributes: dict, program: MbedProgram) -> Config:
     """Assemble config for given target and program directory.
 
     Mbed library and application specific config parameters are parsed from mbed_lib.json and mbed_app.json files
@@ -29,16 +32,36 @@ def assemble_config(target_attributes: dict, search_paths: Iterable[Path], mbed_
 
     Args:
         target_attributes: Mapping of target specific config parameters.
-        search_paths: Iterable of paths to search for mbed_lib.json files.
-        mbed_app_file: The path to mbed_app.json. This can be None.
+        program: MbedProgram to build the config for
     """
     mbed_lib_files: Set[Path] = set()
 
-    for path in search_paths:
+    for path in [program.root, program.mbed_os.root]:
         mbed_lib_files.update(find_files("mbed_lib.json", path.absolute().resolve()))
         mbed_lib_files.update(find_files("mbed_lib.json5", path.absolute().resolve()))
 
-    return _assemble_config_from_sources(target_attributes, list(mbed_lib_files), mbed_app_file)
+    config = _assemble_config_from_sources(target_attributes, list(mbed_lib_files), program.files.app_config_file)
+
+    # Set up the config source path list using the path to every JSON
+    config.json_sources.extend(mbed_lib_files)
+    if program.files.app_config_file is not None:
+        config.json_sources.append(program.files.app_config_file)
+    config.json_sources.append(program.mbed_os.targets_json_file)
+    config.json_sources.append(program.mbed_os.cmsis_mcu_descriptions_json_file)
+    if program.files.custom_targets_json.exists():
+        config.json_sources.append(program.files.custom_targets_json)
+
+    # Make all JSON sources relative paths to the program root
+    def make_relative_if_possible(path: Path):
+        # Sadly, Pathlib did not gain a better way to do this until newer python versions.
+        try:
+            return path.relative_to(program.root)
+        except ValueError:
+            return path
+
+    config.json_sources = [make_relative_if_possible(program.root) for json_source in config.json_sources]
+
+    return config
 
 
 def _assemble_config_from_sources(

tools/python/mbed_tools/build/_internal/config/config.py

@@ -7,6 +7,7 @@
 
 from collections import UserDict
 from typing import Any, Iterable, Hashable, List
+import pathlib
 
 from mbed_tools.build._internal.config.source import Override, ConfigSetting
 
@@ -21,6 +22,11 @@ class Config(UserDict):
     Applies overrides, appends macros, and updates config settings.
     """
 
+    # List of JSON files used to create this config.  Dumped to CMake at the end of configuration
+    # so that it can regenerate configuration if the JSONs change.
+    # All paths will be relative to the Mbed program root directory, or absolute if outside said directory.
+    json_sources: List[pathlib.Path] = []
+
     def __setitem__(self, key: Hashable, item: Any) -> None:
         """Set an item based on its key."""
         if key == CONFIG_SECTION:

tools/python/mbed_tools/build/_internal/templates/mbed_config.tmpl

@@ -6,13 +6,19 @@
 
 include_guard(GLOBAL)
 
-set(MBED_TOOLCHAIN "{{toolchain_name}}" CACHE STRING "")
-set(MBED_TARGET "{{target_name}}" CACHE STRING "")
-set(MBED_CPU_CORE "{{core}}" CACHE STRING "")
-set(MBED_C_LIB "{{c_lib}}" CACHE STRING "")
-set(MBED_PRINTF_LIB "{{printf_lib}}" CACHE STRING "")
-set(MBED_OUTPUT_EXT "{{OUTPUT_EXT}}" CACHE STRING "")
-set(MBED_GREENTEA_TEST_RESET_TIMEOUT "{{forced_reset_timeout}}" CACHE STRING "")
+set(MBED_TOOLCHAIN "{{toolchain_name}}")
+set(MBED_CPU_CORE "{{core}}")
+set(MBED_C_LIB "{{c_lib}}")
+set(MBED_PRINTF_LIB "{{printf_lib}}")
+set(MBED_OUTPUT_EXT "{{OUTPUT_EXT}}")
+set(MBED_GREENTEA_TEST_RESET_TIMEOUT "{{forced_reset_timeout}}")
+
+# JSON files used to generate this config.  If any of these change, the Python config generation
+# scripts must be rerun.
+set(MBED_CONFIG_JSON_SOURCE_FILES {% for json_source in json_sources | sort %}
+    "{{json_source.as_posix()}}"
+{%- endfor %}
+)
 
 list(APPEND MBED_TARGET_SUPPORTED_C_LIBS {% for supported_c_lib in supported_c_libs %}
     {{supported_c_lib}}

tools/python/mbed_tools/build/config.py

@@ -38,7 +38,7 @@ def generate_config(target_name: str, toolchain: str, program: MbedProgram) -> T
     target_build_attributes = get_target_by_name(target_name, targets_data)
     incorporate_memory_bank_data_from_cmsis(target_build_attributes, program)
     config = assemble_config(
-        target_build_attributes, [program.root, program.mbed_os.root], program.files.app_config_file
+        target_build_attributes, program
     )
 
     # Process memory banks and save JSON data for other tools (e.g. memap) to use