From c78065811f7bae031e07bf1d89f391c41025e1d4 Mon Sep 17 00:00:00 2001 From: Mark Inderhees Date: Mon, 29 Sep 2025 16:08:11 -0700 Subject: [PATCH 1/3] build: Dependency handling for syscall gen for changed API Fix issue where changing an existing syscall API does not result in regeneration of syscalls. This causes a build break on incremental builds. The issue comes from an incorrect assumption that file modifications result in updates to folder timestamps on Linux. The fix is to use the same mechanism Windows is using to track file changes, making explicit deps on header files that have syscalls. Signed-off-by: Mark Inderhees --- CMakeLists.txt | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index abe74033e623a..3f6b105e80108 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -788,12 +788,12 @@ set(struct_tags_json ${CMAKE_CURRENT_BINARY_DIR}/misc/generated/struct_tags.json # The syscalls subdirs txt file is constructed by python containing a list of folders to use for # dependency handling, including empty folders. # Windows: The list is used to specify DIRECTORY list with CMAKE_CONFIGURE_DEPENDS attribute. -# Other OS: The list will update whenever a file is added/removed/modified and ensure a re-build. +# Other OS: The list file is updated whenever a directory is added or removed. set(syscalls_subdirs_txt ${CMAKE_CURRENT_BINARY_DIR}/misc/generated/syscalls_subdirs.txt) -# As syscalls_subdirs_txt is updated whenever a file is modified, this file can not be used for -# monitoring of added / removed folders. A trigger file is thus used for correct dependency -# handling. The trigger file will update when a folder is added / removed. +# As syscalls_subdirs_txt is updated only on directory add or remove, this file can not be used for +# monitoring of syscall changes. A trigger file is thus used for correct dependency handling. The +# trigger file will update when syscalls change. set(syscalls_subdirs_trigger ${CMAKE_CURRENT_BINARY_DIR}/misc/generated/syscalls_subdirs.trigger) if(NOT (${CMAKE_HOST_SYSTEM_NAME} STREQUAL Windows)) @@ -812,14 +812,13 @@ execute_process( ) file(STRINGS ${syscalls_subdirs_txt} PARSE_SYSCALLS_PATHS_DEPENDS ENCODING UTF-8) +# Each header file must be monitored as file modifications are not reflected on directory level. +file(GLOB_RECURSE PARSE_SYSCALLS_HEADER_DEPENDS ${ZEPHYR_BASE}/include/*.h) + if(${CMAKE_HOST_SYSTEM_NAME} STREQUAL Windows) # On windows only adding/removing files or folders will be reflected in depends. # Hence adding a file requires CMake to re-run to add this file to the file list. set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${PARSE_SYSCALLS_PATHS_DEPENDS}) - - # Also On Windows each header file must be monitored as file modifications are not reflected - # on directory level. - file(GLOB_RECURSE PARSE_SYSCALLS_HEADER_DEPENDS ${ZEPHYR_BASE}/include/*.h) else() # The syscall parsing depends on the folders in order to detect add/removed/modified files. # When a folder is removed, CMake will try to find a target that creates that dependency. @@ -850,10 +849,10 @@ else() file(WRITE ${syscalls_subdirs_txt} "") endif() - # On other OS'es, modifying a file is reflected on the folder timestamp and hence detected + # On other OS'es, using git checkout is reflected on the folder timestamp and hence detected # when using depend on directory level. # Thus CMake only needs to re-run when sub-directories are added / removed, which is indicated - # using a trigger file. + # by syscalls_subdirs_txt being updated. set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${syscalls_subdirs_txt}) endif() From e9cb4eb440faa4bfa9f52f6dd424670a13c1fc4a Mon Sep 17 00:00:00 2001 From: Mark Inderhees Date: Thu, 2 Oct 2025 14:34:35 -0700 Subject: [PATCH 2/3] build: Track all syscall deps for syscall regen Currently, for syscall generation, cmake is only tracking source files under ${ZEPHYR_BASE}/include. However, source files are actually used from a number of different paths including those provided by clients. This commit adds dependency tracking for all of those sources. Signed-off-by: Mark Inderhees --- CMakeLists.txt | 68 +++++++++++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 28 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3f6b105e80108..4801c02bc8cce 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -801,6 +801,33 @@ if(NOT (${CMAKE_HOST_SYSTEM_NAME} STREQUAL Windows)) endif() # When running CMake it must be ensured that all dependencies are correctly acquired. +# Gather syscall dependencies +list(APPEND SYSCALL_SCAN_DIRS ${ZEPHYR_BASE}/include ${ZEPHYR_BASE}/drivers ${ZEPHYR_BASE}/subsys/net) + +if(CONFIG_APPLICATION_DEFINED_SYSCALL) + list(APPEND SYSCALL_INCLUDE_DIRS ${APPLICATION_SOURCE_DIR}) +endif() + +if(CONFIG_ZTEST) + list(APPEND SYSCALL_INCLUDE_DIRS ${ZEPHYR_BASE}/subsys/testsuite/ztest/include) + + if(CONFIG_NO_OPTIMIZATIONS AND CONFIG_ZTEST_WARN_NO_OPTIMIZATIONS) + message(WARNING "Running tests with CONFIG_NO_OPTIMIZATIONS is generally " + "not supported and known to break in many cases due to stack overflow or " + "other problems. Please do not file issues about it unless the test is " + "specifically tuned to run in this configuration. To disable this warning " + "set CONFIG_ZTEST_WARN_NO_OPTIMIZATIONS=n.") + endif() +endif() + +get_property( + syscalls_include_list + TARGET syscalls_interface + PROPERTY INTERFACE_INCLUDE_DIRECTORIES +) +list(APPEND SYSCALL_INCLUDE_DIRS ${syscalls_include_list}) + +# Walk dependencies to find all subfolders execute_process( COMMAND ${PYTHON_EXECUTABLE} @@ -813,7 +840,12 @@ execute_process( file(STRINGS ${syscalls_subdirs_txt} PARSE_SYSCALLS_PATHS_DEPENDS ENCODING UTF-8) # Each header file must be monitored as file modifications are not reflected on directory level. -file(GLOB_RECURSE PARSE_SYSCALLS_HEADER_DEPENDS ${ZEPHYR_BASE}/include/*.h) +foreach(d IN LISTS SYSCALL_SCAN_DIRS SYSCALL_INCLUDE_DIRS) + list(APPEND glob_syscalls_headers + ${d}/*.h + ) +endforeach() +file(GLOB_RECURSE PARSE_SYSCALLS_HEADER_DEPENDS ${glob_syscalls_headers}) if(${CMAKE_HOST_SYSTEM_NAME} STREQUAL Windows) # On windows only adding/removing files or folders will be reflected in depends. @@ -856,30 +888,12 @@ else() set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${syscalls_subdirs_txt}) endif() -# syscall declarations are searched for in the SYSCALL_INCLUDE_DIRS -if(CONFIG_APPLICATION_DEFINED_SYSCALL) - list(APPEND SYSCALL_INCLUDE_DIRS ${APPLICATION_SOURCE_DIR}) -endif() - -if(CONFIG_ZTEST) - list(APPEND SYSCALL_INCLUDE_DIRS ${ZEPHYR_BASE}/subsys/testsuite/ztest/include) - - if(CONFIG_NO_OPTIMIZATIONS AND CONFIG_ZTEST_WARN_NO_OPTIMIZATIONS) - message(WARNING "Running tests with CONFIG_NO_OPTIMIZATIONS is generally " - "not supported and known to break in many cases due to stack overflow or " - "other problems. Please do not file issues about it unless the test is " - "specifically tuned to run in this configuration. To disable this warning " - "set CONFIG_ZTEST_WARN_NO_OPTIMIZATIONS=n.") - endif() - -endif() - -get_property( - syscalls_include_list - TARGET syscalls_interface - PROPERTY INTERFACE_INCLUDE_DIRECTORIES -) -list(APPEND SYSCALL_INCLUDE_DIRS ${syscalls_include_list}) +# syscall declarations are searched for in the SYSCALL_SCAN_DIRS and SYSCALL_INCLUDE_DIRS +foreach(d ${SYSCALL_SCAN_DIRS}) + list(APPEND parse_syscalls_scan_args + --scan ${d} + ) +endforeach() foreach(d ${SYSCALL_INCLUDE_DIRS}) list(APPEND parse_syscalls_include_args @@ -894,9 +908,7 @@ add_custom_command( COMMAND ${PYTHON_EXECUTABLE} ${ZEPHYR_BASE}/scripts/build/parse_syscalls.py - --scan ${ZEPHYR_BASE}/include # Read files from this dir - --scan ${ZEPHYR_BASE}/drivers # For net sockets - --scan ${ZEPHYR_BASE}/subsys/net # More net sockets + ${parse_syscalls_scan_args} # Search for syscall declarations in these dirs ${parse_syscalls_include_args} # Read files from these dirs also --json-file ${syscalls_json} # Write this file --tag-struct-file ${struct_tags_json} # Write subsystem list to this file From 1d30831de93b410afe55449ef8713d36cdd02f31 Mon Sep 17 00:00:00 2001 From: Mark Inderhees Date: Thu, 2 Oct 2025 15:22:06 -0700 Subject: [PATCH 3/3] build: Track all folders for syscall gen Currently, for syscall generation, cmake is only tracking folders under ${ZEPHYR_BASE}/include. However, folders are actually used from a number of different paths including those provided by clients. This commit adds dependency tracking for all of those sources. Signed-off-by: Mark Inderhees --- CMakeLists.txt | 10 +++- scripts/build/subfolder_list.py | 83 +++++++++++++++++++++------------ 2 files changed, 61 insertions(+), 32 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4801c02bc8cce..87c8daa4bfce9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -828,11 +828,17 @@ get_property( list(APPEND SYSCALL_INCLUDE_DIRS ${syscalls_include_list}) # Walk dependencies to find all subfolders +foreach(d IN LISTS SYSCALL_SCAN_DIRS SYSCALL_INCLUDE_DIRS) + list(APPEND syscalls_subfolders + --directory ${d} + ) +endforeach() + execute_process( COMMAND ${PYTHON_EXECUTABLE} ${ZEPHYR_BASE}/scripts/build/subfolder_list.py - --directory ${ZEPHYR_BASE}/include # Walk this directory + ${syscalls_subfolders} # Walk these directories --out-file ${syscalls_subdirs_txt} # Write file with discovered folder --trigger-file ${syscalls_subdirs_trigger} # Trigger file that is used for json generation ${syscalls_links} # If defined, create symlinks for dependencies @@ -869,7 +875,7 @@ else() COMMAND ${PYTHON_EXECUTABLE} ${ZEPHYR_BASE}/scripts/build/subfolder_list.py - --directory ${ZEPHYR_BASE}/include # Walk this directory + ${syscalls_subfolders} # Walk these directories --out-file ${syscalls_subdirs_txt} # Write file with discovered folder --trigger-file ${syscalls_subdirs_trigger} # Trigger file that is used for json generation ${syscalls_links} # If defined, create symlinks for dependencies diff --git a/scripts/build/subfolder_list.py b/scripts/build/subfolder_list.py index 2570ebe8534da..0bcdf1fed7127 100755 --- a/scripts/build/subfolder_list.py +++ b/scripts/build/subfolder_list.py @@ -21,8 +21,9 @@ def parse_args(): formatter_class=argparse.RawDescriptionHelpFormatter, allow_abbrev=False) - parser.add_argument('-d', '--directory', required=True, - help='Directory to walk for sub-directory discovery') + parser.add_argument('-d', '--directory', required=True, action='append', + help='Directory to walk for sub-directory discovery \ + (can be specified multiple times)') parser.add_argument('-c', '--create-links', required=False, help='Create links for each directory found in \ directory given') @@ -37,34 +38,55 @@ def parse_args(): return args -def get_subfolder_list(directory, create_links=None): - """Return subfolder list of a directory""" +def get_subfolder_list(directories, create_links=None): + """Return subfolder list of directories""" dirlist = [] - - if create_links is not None: - if not os.path.exists(create_links): - os.makedirs(create_links) - symbase = os.path.basename(directory) - symlink = create_links + os.path.sep + symbase - if not os.path.exists(symlink): - os.symlink(directory, symlink) - dirlist.append(symlink) - else: - dirlist.append(directory) - - for root, dirs, _ in os.walk(directory, topdown=True): - dirs.sort() - for subdir in dirs: - if create_links is not None: - targetdirectory = os.path.join(root, subdir) - reldir = os.path.relpath(targetdirectory, directory) - linkname = symbase + '_' + reldir.replace(os.path.sep, '_') - symlink = create_links + os.path.sep + linkname - if not os.path.exists(symlink): - os.symlink(targetdirectory, symlink) - dirlist.append(symlink) - else: - dirlist.append(os.path.join(root, subdir)) + used_link_names = set() + + for directory in directories: + if create_links is not None: + if not os.path.exists(create_links): + os.makedirs(create_links) + symbase = os.path.basename(directory) + + # Ensure unique link name for the base directory + linkname = symbase + counter = 1 + while linkname in used_link_names: + linkname = f"{symbase}_{counter}" + counter += 1 + used_link_names.add(linkname) + + symlink = create_links + os.path.sep + linkname + if not os.path.exists(symlink): + os.symlink(directory, symlink) + dirlist.append(symlink) + else: + dirlist.append(directory) + + for root, dirs, _ in os.walk(directory, topdown=True): + dirs.sort() + for subdir in dirs: + if create_links is not None: + targetdirectory = os.path.join(root, subdir) + reldir = os.path.relpath(targetdirectory, directory) + base_linkname = (os.path.basename(directory) + + '_' + reldir.replace(os.path.sep, '_')) + + # Ensure unique link name for subdirectories + linkname = base_linkname + counter = 1 + while linkname in used_link_names: + linkname = f"{base_linkname}_{counter}" + counter += 1 + used_link_names.add(linkname) + + symlink = create_links + os.path.sep + linkname + if not os.path.exists(symlink): + os.symlink(targetdirectory, symlink) + dirlist.append(symlink) + else: + dirlist.append(os.path.join(root, subdir)) return dirlist @@ -108,7 +130,8 @@ def main(): """Parse command line arguments and take respective actions""" args = parse_args() - dirs = get_subfolder_list(args.directory, args.create_links) + directories = sorted(set(args.directory)) + dirs = get_subfolder_list(directories, args.create_links) gen_out_file(args.out_file, dirs) # Always touch trigger file to ensure json files are updated