build: Track all syscall deps

CMakeLists.txt

@@ -788,38 +788,75 @@ set(struct_tags_json ${CMAKE_CURRENT_BINARY_DIR}/misc/generated/struct_tags.json
 # The syscalls subdirs txt file is constructed by python containing a list of folders to use for
 # dependency handling, including empty folders.
 # Windows:  The list is used to specify DIRECTORY list with CMAKE_CONFIGURE_DEPENDS attribute.
-# Other OS: The list will update whenever a file is added/removed/modified and ensure a re-build.
+# Other OS: The list file is updated whenever a directory is added or removed.
 set(syscalls_subdirs_txt ${CMAKE_CURRENT_BINARY_DIR}/misc/generated/syscalls_subdirs.txt)
 
-# As syscalls_subdirs_txt is updated whenever a file is modified, this file can not be used for
-# monitoring of added / removed folders. A trigger file is thus used for correct dependency
-# handling. The trigger file will update when a folder is added / removed.
+# As syscalls_subdirs_txt is updated only on directory add or remove, this file can not be used for
+# monitoring of syscall changes. A trigger file is thus used for correct dependency handling. The
+# trigger file will update when syscalls change.
 set(syscalls_subdirs_trigger ${CMAKE_CURRENT_BINARY_DIR}/misc/generated/syscalls_subdirs.trigger)
 
 if(NOT (${CMAKE_HOST_SYSTEM_NAME} STREQUAL Windows))
   set(syscalls_links --create-links ${CMAKE_CURRENT_BINARY_DIR}/misc/generated/syscalls_links)
 endif()
 
 # When running CMake it must be ensured that all dependencies are correctly acquired.
+# Gather syscall dependencies
+list(APPEND SYSCALL_SCAN_DIRS ${ZEPHYR_BASE}/include ${ZEPHYR_BASE}/drivers ${ZEPHYR_BASE}/subsys/net)
+
+if(CONFIG_APPLICATION_DEFINED_SYSCALL)
+  list(APPEND SYSCALL_INCLUDE_DIRS ${APPLICATION_SOURCE_DIR})
+endif()
+
+if(CONFIG_ZTEST)
+  list(APPEND SYSCALL_INCLUDE_DIRS ${ZEPHYR_BASE}/subsys/testsuite/ztest/include)
+
+  if(CONFIG_NO_OPTIMIZATIONS AND CONFIG_ZTEST_WARN_NO_OPTIMIZATIONS)
+    message(WARNING "Running tests with CONFIG_NO_OPTIMIZATIONS is generally "
+    "not supported and known to break in many cases due to stack overflow or "
+    "other problems. Please do not file issues about it unless the test is "
+    "specifically tuned to run in this configuration. To disable this warning "
+    "set CONFIG_ZTEST_WARN_NO_OPTIMIZATIONS=n.")
+  endif()
+endif()
+
+get_property(
+  syscalls_include_list
+  TARGET syscalls_interface
+  PROPERTY INTERFACE_INCLUDE_DIRECTORIES
+)
+list(APPEND SYSCALL_INCLUDE_DIRS ${syscalls_include_list})
+
+# Walk dependencies to find all subfolders
+foreach(d IN LISTS SYSCALL_SCAN_DIRS SYSCALL_INCLUDE_DIRS)
+  list(APPEND syscalls_subfolders
+    --directory ${d}
+    )
+endforeach()
+
 execute_process(
   COMMAND
   ${PYTHON_EXECUTABLE}
   ${ZEPHYR_BASE}/scripts/build/subfolder_list.py
-  --directory        ${ZEPHYR_BASE}/include      # Walk this directory
+  ${syscalls_subfolders}                         # Walk these directories
   --out-file         ${syscalls_subdirs_txt}     # Write file with discovered folder
   --trigger-file     ${syscalls_subdirs_trigger} # Trigger file that is used for json generation
   ${syscalls_links}                              # If defined, create symlinks for dependencies
 )
 file(STRINGS ${syscalls_subdirs_txt} PARSE_SYSCALLS_PATHS_DEPENDS ENCODING UTF-8)
 
+# Each header file must be monitored as file modifications are not reflected on directory level.
+foreach(d IN LISTS SYSCALL_SCAN_DIRS SYSCALL_INCLUDE_DIRS)
+  list(APPEND glob_syscalls_headers
+    ${d}/*.h
+    )
+endforeach()
+file(GLOB_RECURSE PARSE_SYSCALLS_HEADER_DEPENDS ${glob_syscalls_headers})
+
 if(${CMAKE_HOST_SYSTEM_NAME} STREQUAL Windows)
   # On windows only adding/removing files or folders will be reflected in depends.
   # Hence adding a file requires CMake to re-run to add this file to the file list.
   set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${PARSE_SYSCALLS_PATHS_DEPENDS})
-
-  # Also On Windows each header file must be monitored as file modifications are not reflected
-  # on directory level.
-  file(GLOB_RECURSE PARSE_SYSCALLS_HEADER_DEPENDS ${ZEPHYR_BASE}/include/*.h)
 else()
   # The syscall parsing depends on the folders in order to detect add/removed/modified files.
   # When a folder is removed, CMake will try to find a target that creates that dependency.
@@ -838,7 +875,7 @@ else()
     COMMAND
     ${PYTHON_EXECUTABLE}
     ${ZEPHYR_BASE}/scripts/build/subfolder_list.py
-    --directory        ${ZEPHYR_BASE}/include      # Walk this directory
+    ${syscalls_subfolders}                         # Walk these directories
     --out-file         ${syscalls_subdirs_txt}     # Write file with discovered folder
     --trigger-file     ${syscalls_subdirs_trigger} # Trigger file that is used for json generation
     ${syscalls_links}                              # If defined, create symlinks for dependencies
@@ -850,37 +887,19 @@ else()
     file(WRITE ${syscalls_subdirs_txt} "")
   endif()
 
-  # On other OS'es, modifying a file is reflected on the folder timestamp and hence detected
+  # On other OS'es, using git checkout is reflected on the folder timestamp and hence detected
   # when using depend on directory level.
   # Thus CMake only needs to re-run when sub-directories are added / removed, which is indicated
-  # using a trigger file.
+  # by syscalls_subdirs_txt being updated.
   set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${syscalls_subdirs_txt})
 endif()
 
-# syscall declarations are searched for in the SYSCALL_INCLUDE_DIRS
-if(CONFIG_APPLICATION_DEFINED_SYSCALL)
-  list(APPEND SYSCALL_INCLUDE_DIRS ${APPLICATION_SOURCE_DIR})
-endif()
-
-if(CONFIG_ZTEST)
-  list(APPEND SYSCALL_INCLUDE_DIRS ${ZEPHYR_BASE}/subsys/testsuite/ztest/include)
-
-  if(CONFIG_NO_OPTIMIZATIONS AND CONFIG_ZTEST_WARN_NO_OPTIMIZATIONS)
-    message(WARNING "Running tests with CONFIG_NO_OPTIMIZATIONS is generally "
-    "not supported and known to break in many cases due to stack overflow or "
-    "other problems. Please do not file issues about it unless the test is "
-    "specifically tuned to run in this configuration. To disable this warning "
-    "set CONFIG_ZTEST_WARN_NO_OPTIMIZATIONS=n.")
-  endif()
-
-endif()
-
-get_property(
-  syscalls_include_list
-  TARGET syscalls_interface
-  PROPERTY INTERFACE_INCLUDE_DIRECTORIES
-)
-list(APPEND SYSCALL_INCLUDE_DIRS ${syscalls_include_list})
+# syscall declarations are searched for in the SYSCALL_SCAN_DIRS and SYSCALL_INCLUDE_DIRS
+foreach(d ${SYSCALL_SCAN_DIRS})
+  list(APPEND parse_syscalls_scan_args
+    --scan ${d}
+    )
+endforeach()
 
 foreach(d ${SYSCALL_INCLUDE_DIRS})
   list(APPEND parse_syscalls_include_args
@@ -895,9 +914,7 @@ add_custom_command(
   COMMAND
   ${PYTHON_EXECUTABLE}
   ${ZEPHYR_BASE}/scripts/build/parse_syscalls.py
-  --scan             ${ZEPHYR_BASE}/include         # Read files from this dir
-  --scan             ${ZEPHYR_BASE}/drivers         # For net sockets
-  --scan             ${ZEPHYR_BASE}/subsys/net      # More net sockets
+  ${parse_syscalls_scan_args}                       # Search for syscall declarations in these dirs
   ${parse_syscalls_include_args}                    # Read files from these dirs also
   --json-file        ${syscalls_json}               # Write this file
   --tag-struct-file  ${struct_tags_json}            # Write subsystem list to this file

scripts/build/subfolder_list.py

@@ -1,4 +1,4 @@
 #!/usr/bin/env python3
 # SPDX-License-Identifier: Apache-2.0
 
 """Write subfolder list to a file
@@ -21,8 +21,9 @@
         formatter_class=argparse.RawDescriptionHelpFormatter,
         allow_abbrev=False)
 
-    parser.add_argument('-d', '--directory', required=True,
-                        help='Directory to walk for sub-directory discovery')
+    parser.add_argument('-d', '--directory', required=True, action='append',
+                        help='Directory to walk for sub-directory discovery \
+                              (can be specified multiple times)')
     parser.add_argument('-c', '--create-links', required=False,
                         help='Create links for each directory found in \
                               directory given')
@@ -37,34 +38,55 @@
     return args
 
 
-def get_subfolder_list(directory, create_links=None):
-    """Return subfolder list of a directory"""
+def get_subfolder_list(directories, create_links=None):
+    """Return subfolder list of directories"""
     dirlist = []
-
-    if create_links is not None:
-        if not os.path.exists(create_links):
-            os.makedirs(create_links)
-        symbase = os.path.basename(directory)
-        symlink = create_links + os.path.sep + symbase
-        if not os.path.exists(symlink):
-            os.symlink(directory, symlink)
-        dirlist.append(symlink)
-    else:
-        dirlist.append(directory)
-
-    for root, dirs, _ in os.walk(directory, topdown=True):
-        dirs.sort()
-        for subdir in dirs:
-            if create_links is not None:
-                targetdirectory = os.path.join(root, subdir)
-                reldir = os.path.relpath(targetdirectory, directory)
-                linkname = symbase + '_' + reldir.replace(os.path.sep, '_')
-                symlink = create_links + os.path.sep + linkname
-                if not os.path.exists(symlink):
-                    os.symlink(targetdirectory, symlink)
-                dirlist.append(symlink)
-            else:
-                dirlist.append(os.path.join(root, subdir))
+    used_link_names = set()
+
+    for directory in directories:
+        if create_links is not None:
+            if not os.path.exists(create_links):
+                os.makedirs(create_links)
+            symbase = os.path.basename(directory)
+
+            # Ensure unique link name for the base directory
+            linkname = symbase
+            counter = 1
+            while linkname in used_link_names:
+                linkname = f"{symbase}_{counter}"
+                counter += 1
+            used_link_names.add(linkname)
+
+            symlink = create_links + os.path.sep + linkname
+            if not os.path.exists(symlink):
+                os.symlink(directory, symlink)
+            dirlist.append(symlink)
+        else:
+            dirlist.append(directory)
+
+        for root, dirs, _ in os.walk(directory, topdown=True):
+            dirs.sort()
+            for subdir in dirs:
+                if create_links is not None:
+                    targetdirectory = os.path.join(root, subdir)
+                    reldir = os.path.relpath(targetdirectory, directory)
+                    base_linkname = (os.path.basename(directory) +
+                        '_' + reldir.replace(os.path.sep, '_'))
+
+                    # Ensure unique link name for subdirectories
+                    linkname = base_linkname
+                    counter = 1
+                    while linkname in used_link_names:
+                        linkname = f"{base_linkname}_{counter}"
+                        counter += 1
+                    used_link_names.add(linkname)
+
+                    symlink = create_links + os.path.sep + linkname
+                    if not os.path.exists(symlink):
+                        os.symlink(targetdirectory, symlink)
+                    dirlist.append(symlink)
+                else:
+                    dirlist.append(os.path.join(root, subdir))
 
     return dirlist
 
@@ -108,7 +130,8 @@
     """Parse command line arguments and take respective actions"""
     args = parse_args()
 
-    dirs = get_subfolder_list(args.directory, args.create_links)
+    directories = sorted(set(args.directory))
+    dirs = get_subfolder_list(directories, args.create_links)
     gen_out_file(args.out_file, dirs)
 
     # Always touch trigger file to ensure json files are updated