fix: Optionally exclude directories from list_files_under for performance (#1488)

This PR adds an `exclude_directories` argument to
`repository_files.list_files_under` which resulted in a massive
performance improvement for packages with lots of resources.

I noticed that it was taking a really long time to "fetch" a package
that was local to my machine, so there was no actual network operations
required.

I dug into the issue and it's actually from a change I added in [this
PR](#1039)
to expand certain types of resources into a list of their assets. The
overwhelming majority of the time was being spent on the `if not
repository_files.is_directory(repository_ctx, p)` check inside of the
loop. By omitting directories from the result of `list_files_under` we
can skip all of that work.

### Before - 225s
<img width="1543" alt="image"
src="https://github.com/user-attachments/assets/67f6f5e2-66b7-4143-95fd-8401ea6c8347"
/>

### After - 2s
<img width="1300" alt="image"
src="https://github.com/user-attachments/assets/396c6eb2-4b59-41ab-811d-3e10bacba243"
/>

I didn't see any uses of `list_files_under` that were relying on it to
contain directories. Should we change the behavior to never include
directories?

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

Author: jflan-dd

swiftpkg/internal/pkginfos.bzl

@@ -309,15 +309,12 @@ def _new_target_from_json_maps(
     for directory in resource_directories:
         sets.remove(resources_set, directory)
 
-        resource_files = [
-            p
-            for p in repository_files.list_files_under(
-                repository_ctx,
-                directory.path,
-                exclude_paths = exclude_paths,
-            )
-            if not repository_files.is_directory(repository_ctx, p)
-        ]
+        resource_files = repository_files.list_files_under(
+            repository_ctx,
+            directory.path,
+            exclude_paths = exclude_paths,
+            exclude_directories = True,
+        )
 
         for p in resource_files:
             res = _new_resource_from_discovered_resource(p)
@@ -1023,23 +1020,14 @@ def _new_swift_src_info_from_sources(
         repository_ctx,
         target_path,
         exclude_paths = exclude_paths,
+        exclude_directories = True,
     )
 
-    # Identify the directories
-    directories = repository_files.list_directories_under(
-        repository_ctx,
-        target_path,
-        exclude_paths = exclude_paths,
-    )
-    dirs_set = sets.make(directories)
-
     # The paths should be relative to the target not the root of the workspace.
-    # Do not include directories in the output.
     discovered_res_files = [
         f
         for f in all_target_files
-        if not sets.contains(dirs_set, f) and
-           resource_files.is_auto_discovered_resource(f)
+        if resource_files.is_auto_discovered_resource(f)
     ]
 
     return _new_swift_src_info(

swiftpkg/internal/repository_files.bzl

@@ -65,6 +65,7 @@ def _list_files_under(
         repository_ctx,
         path,
         exclude_paths = [],
+        exclude_directories = False,
         by_name = None,
         depth = None):
     """Retrieves the list of files under the specified path.
@@ -76,6 +77,7 @@ def _list_files_under(
         path: A path `string` value.
         exclude_paths: Optional. A `list` of path `string` values that should be
             excluded from the result.
+        exclude_directories: Optional. Exclude directories from the result.
         by_name: Optional. The name pattern that must be matched.
         depth: Optional. The depth as an `int` at which the directory must
             live from the starting path.
@@ -95,6 +97,8 @@ def _list_files_under(
         find_args.extend(["-mindepth", depth_str, "-maxdepth", depth_str])
     if by_name != None:
         find_args.extend(["-name", by_name])
+    if exclude_directories:
+        find_args.extend(["-not", "-type", "d"])
     exec_result = repository_ctx.execute(find_args, quiet = True)
     if exec_result.return_code != 0:
         fail("Failed to list files in %s. stderr:\n%s" % (path, exec_result.stderr))