Improve performance of excluded files filter

The current algorithm is like "collect all included files and subtract all excluded files".
Collecting all included and all excluded files relies on the file system. This can become slow
when the patterns used to exclude files resolve to a large number of files.

The new approach only collects all lintable files and checks them against the exclude patterns.
This can be done by in-memory string-regex-match and does therefore not require file system accesses.

(cherry picked from commit 152355e)

Author: SimplyDanny

BUILD

@@ -70,6 +70,7 @@ swift_library(
     visibility = ["//visibility:public"],
     deps = [
         ":Yams.wrapper",
+        "@FilenameMatcher",
         "@SourceKittenFramework",
         "@SwiftSyntax//:SwiftIDEUtils_opt",
         "@SwiftSyntax//:SwiftOperators_opt",

CHANGELOG.md

@@ -689,6 +689,7 @@
 * Ignore TipKit's `#Rule` macro in `empty_count` rule.  
   [Ueeek](https://github.com/Ueeek)
   [#5883](https://github.com/realm/SwiftLint/issues/5883)
+#### Bug Fixes
 
 * Ignore super calls with trailing closures in `unneeded_override` rule.  
   [SimplyDanny](https://github.com/SimplyDanny)

MODULE.bazel

@@ -15,6 +15,7 @@ bazel_dep(name = "rules_swift", version = "3.1.2", max_compatibility_level = 3,
 
 bazel_dep(name = "sourcekitten", version = "0.37.2", repo_name = "SourceKittenFramework")
 bazel_dep(name = "swift_argument_parser", version = "1.6.2", repo_name = "SwiftArgumentParser")
+bazel_dep(name = "swift-filename-matcher", version = "2.0.1", repo_name = "FilenameMatcher")
 bazel_dep(name = "yams", version = "6.2.0", repo_name = "Yams")
 
 swiftlint_repos = use_extension("//bazel:repos.bzl", "swiftlint_repos_bzlmod")

Package.resolved

@@ -42,6 +42,7 @@ let package = Package(
         .package(url: "https://github.com/scottrhoyt/SwiftyTextTable.git", .upToNextMajor(from: "0.9.0")),
         .package(url: "https://github.com/JohnSundell/CollectionConcurrencyKit.git", .upToNextMajor(from: "0.2.0")),
         .package(url: "https://github.com/krzyzanowskim/CryptoSwift.git", .upToNextMajor(from: "1.9.0")),
+        .package(url: "https://github.com/ileitch/swift-filename-matcher", .upToNextMinor(from: "2.0.1")),
     ],
     targets: [
         .executableTarget(
@@ -65,6 +66,7 @@ let package = Package(
         .target(
             name: "SwiftLintFramework",
             dependencies: [
+                .product(name: "FilenameMatcher", package: "swift-filename-matcher"),
                 "SwiftLintBuiltInRules",
                 "SwiftLintCore",
                 "SwiftLintExtraRules",
@@ -96,6 +98,7 @@ let package = Package(
             dependencies: [
                 .product(name: "CryptoSwift", package: "CryptoSwift", condition: .when(platforms: [.linux])),
                 .target(name: "DyldWarningWorkaround", condition: .when(platforms: [.macOS])),
+                .product(name: "FilenameMatcher", package: "swift-filename-matcher"),
                 .product(name: "SourceKittenFramework", package: "SourceKitten"),
                 .product(name: "SwiftIDEUtils", package: "swift-syntax"),
                 .product(name: "SwiftOperators", package: "swift-syntax"),
@@ -167,6 +170,7 @@ let package = Package(
         .testTarget(
             name: "FileSystemAccessTests",
             dependencies: [
+                .product(name: "FilenameMatcher", package: "swift-filename-matcher"),
                 "SwiftLintFramework",
                 "TestHelpers",
                 "SwiftLintCoreMacros",

Package.swift

@@ -68,6 +68,9 @@ public extension String {
 
     /// Returns a new string, converting the path to a canonical absolute path.
     ///
+    /// > Important: This method might use an incorrect working directory internally. This can cause test failures
+    /// in Bazel builds but does not seem to cause trouble in production.
+    ///
     /// - returns: A new `String`.
     func absolutePathStandardized() -> String {
         bridge().absolutePathRepresentation().bridge().standardizingPath

Source/SwiftLintCore/Extensions/String+SwiftLint.swift

@@ -252,15 +252,12 @@ extension Configuration {
             guard options.forceExclude else {
                 return files
             }
-
             let scriptInputPaths = files.compactMap(\.path)
-
-            if options.useExcludingByPrefix {
-                return filterExcludedPathsByPrefix(in: scriptInputPaths)
-                    .map(SwiftLintFile.init(pathDeferringReading:))
-            }
-            return filterExcludedPaths(excludedPaths(), in: scriptInputPaths)
-                .map(SwiftLintFile.init(pathDeferringReading:))
+            return (
+                visitor.options.useExcludingByPrefix
+                    ? filterExcludedPathsByPrefix(in: scriptInputPaths)
+                    : filterExcludedPaths(in: scriptInputPaths)
+            ).map(SwiftLintFile.init(pathDeferringReading:))
         }
         if !options.quiet {
             let filesInfo: String
@@ -272,14 +269,12 @@ extension Configuration {
 
             queuedPrintError("\(options.capitalizedVerb) Swift files \(filesInfo)")
         }
-        let excludeLintableFilesBy = options.useExcludingByPrefix
-            ? Configuration.ExcludeBy.prefix
-            : .paths(excludedPaths: excludedPaths())
-        return options.paths.flatMap {
+        return visitor.options.paths.flatMap {
             self.lintableFiles(
                 inPath: $0,
-                forceExclude: options.forceExclude,
-                excludeBy: excludeLintableFilesBy)
+                forceExclude: visitor.options.forceExclude,
+                excludeByPrefix: visitor.options.useExcludingByPrefix
+            )
         }
     }
 

Source/SwiftLintFramework/Configuration+CommandLine.swift

@@ -1,25 +1,22 @@
+import FilenameMatcher
 import Foundation
 
 extension Configuration {
-    public enum ExcludeBy {
-        case prefix
-        case paths(excludedPaths: [String])
-    }
-
     // MARK: Lintable Paths
+
     /// Returns the files that can be linted by SwiftLint in the specified parent path.
     ///
     /// - parameter path:            The parent path in which to search for lintable files. Can be a directory or a
     ///                              file.
     /// - parameter forceExclude:    Whether or not excludes defined in this configuration should be applied even if
     ///                              `path` is an exact match.
-    /// - parameter excludeByPrefix: Whether or not uses excluding by prefix algorithm.
+    /// - parameter excludeByPrefix: Whether or not it uses the exclude-by-prefix algorithm.
     ///
     /// - returns: Files to lint.
     public func lintableFiles(inPath path: String,
                               forceExclude: Bool,
-                              excludeBy: ExcludeBy) -> [SwiftLintFile] {
-        lintablePaths(inPath: path, forceExclude: forceExclude, excludeBy: excludeBy)
+                              excludeByPrefix: Bool) -> [SwiftLintFile] {
+        lintablePaths(inPath: path, forceExclude: forceExclude, excludeByPrefix: excludeByPrefix)
             .parallelCompactMap {
                 SwiftLintFile(pathDeferringReading: $0)
             }
@@ -31,63 +28,55 @@ extension Configuration {
     ///                              file.
     /// - parameter forceExclude:    Whether or not excludes defined in this configuration should be applied even if
     ///                              `path` is an exact match.
-    /// - parameter excludeByPrefix: Whether or not uses excluding by prefix algorithm.
+    /// - parameter excludeByPrefix: Whether or not it uses the exclude-by-prefix algorithm.
     /// - parameter fileManager:     The lintable file manager to use to search for lintable files.
     ///
     /// - returns: Paths for files to lint.
     internal func lintablePaths(
         inPath path: String,
         forceExclude: Bool,
-        excludeBy: ExcludeBy,
+        excludeByPrefix: Bool,
         fileManager: some LintableFileManager = FileManager.default
     ) -> [String] {
         if fileManager.isFile(atPath: path) {
+            let file = fileManager.filesToLint(inPath: path, rootDirectory: nil)
             if forceExclude {
-                switch excludeBy {
-                case .prefix:
-                    return filterExcludedPathsByPrefix(in: [path.absolutePathStandardized()])
-                case .paths(let excludedPaths):
-                    return filterExcludedPaths(excludedPaths, in: [path.absolutePathStandardized()])
-                }
+                return excludeByPrefix
+                    ? filterExcludedPathsByPrefix(in: file)
+                    : filterExcludedPaths(in: file)
             }
             // If path is a file and we're not forcing excludes, skip filtering with excluded/included paths
-            return [path]
+            return file
         }
 
         let pathsForPath = includedPaths.isEmpty ? fileManager.filesToLint(inPath: path, rootDirectory: nil) : []
         let includedPaths = self.includedPaths
             .flatMap(Glob.resolveGlob)
             .parallelFlatMap { fileManager.filesToLint(inPath: $0, rootDirectory: rootDirectory) }
 
-        switch excludeBy {
-        case .prefix:
-            return filterExcludedPathsByPrefix(in: pathsForPath, includedPaths)
-        case .paths(let excludedPaths):
-            return filterExcludedPaths(excludedPaths, in: pathsForPath, includedPaths)
-        }
+        return excludeByPrefix
+            ? filterExcludedPathsByPrefix(in: pathsForPath + includedPaths)
+            : filterExcludedPaths(in: pathsForPath + includedPaths)
     }
 
     /// Returns an array of file paths after removing the excluded paths as defined by this configuration.
     ///
-    /// - parameter fileManager: The lintable file manager to use to expand the excluded paths into all matching paths.
-    /// - parameter paths:       The input paths to filter.
+    /// - parameter paths: The input paths to filter.
     ///
     /// - returns: The input paths after removing the excluded paths.
-    public func filterExcludedPaths(
-        _ excludedPaths: [String],
-        in paths: [String]...
-    ) -> [String] {
-        let allPaths = paths.flatMap(\.self)
+    public func filterExcludedPaths(in paths: [String]) -> [String] {
         #if os(Linux)
-        let result = NSMutableOrderedSet(capacity: allPaths.count)
-        result.addObjects(from: allPaths)
+        let result = NSMutableOrderedSet(capacity: paths.count)
+        result.addObjects(from: paths)
         #else
-        let result = NSMutableOrderedSet(array: allPaths)
+        let result = NSOrderedSet(array: paths)
         #endif
-
-        result.minusSet(Set(excludedPaths))
-        // swiftlint:disable:next force_cast
-        return result.map { $0 as! String }
+        let exclusionPatterns = self.excludedPaths.flatMap {
+            Glob.createFilenameMatchers(root: rootDirectory, pattern: $0)
+        }
+        return result.array
+            .parallelCompactMap { exclusionPatterns.anyMatch(filename: $0 as! String) ? nil : $0 as? String }
+            // swiftlint:disable:previous force_cast
     }
 
     /// Returns the file paths that are excluded by this configuration using filtering by absolute path prefix.
@@ -96,25 +85,12 @@ extension Configuration {
     /// algorithm `filterExcludedPaths`.
     ///
     /// - returns: The input paths after removing the excluded paths.
-    public func filterExcludedPathsByPrefix(in paths: [String]...) -> [String] {
-        let allPaths = paths.flatMap(\.self)
+    public func filterExcludedPathsByPrefix(in paths: [String]) -> [String] {
         let excludedPaths = self.excludedPaths
-            .parallelFlatMap { @Sendable in Glob.resolveGlob($0) }
+            .parallelFlatMap { Glob.resolveGlob($0) }
             .map { $0.absolutePathStandardized() }
-        return allPaths.filter { path in
+        return paths.filter { path in
             !excludedPaths.contains { path.hasPrefix($0) }
         }
     }
-
-    /// Returns the file paths that are excluded by this configuration after expanding them using the specified file
-    /// manager.
-    ///
-    /// - parameter fileManager: The file manager to get child paths in a given parent location.
-    ///
-    /// - returns: The expanded excluded file paths.
-    public func excludedPaths(fileManager: some LintableFileManager = FileManager.default) -> [String] {
-        excludedPaths
-            .flatMap(Glob.resolveGlob)
-            .parallelFlatMap { fileManager.filesToLint(inPath: $0, rootDirectory: rootDirectory) }
-    }
 }

Source/SwiftLintFramework/Configuration/Configuration+LintableFiles.swift

@@ -1,3 +1,4 @@
+import FilenameMatcher
 import Foundation
 import SourceKittenFramework
 
@@ -37,6 +38,29 @@ struct Glob {
             .map { $0.absolutePathStandardized() }
     }
 
+    static func createFilenameMatchers(root: String, pattern: String) -> [FilenameMatcher] {
+        var absolutPathPattern = pattern
+        if !pattern.starts(with: root) {
+            // If the root is not already part of the pattern, prepend it.
+            absolutPathPattern = root + (root.hasSuffix("/") ? "" : "/") + absolutPathPattern
+        }
+        absolutPathPattern = absolutPathPattern.absolutePathStandardized()
+        if pattern.hasSuffix(".swift") || pattern.hasSuffix("/**") {
+            // Suffix is already well defined.
+            return [FilenameMatcher(pattern: absolutPathPattern)]
+        }
+        if pattern.hasSuffix("/") {
+            // Matching all files in the folder.
+            return [FilenameMatcher(pattern: absolutPathPattern + "**")]
+        }
+        // The pattern could match files in the last folder in the path or all contained files if the last component
+        // represents folders.
+        return [
+            FilenameMatcher(pattern: absolutPathPattern),
+            FilenameMatcher(pattern: absolutPathPattern + "/**"),
+        ]
+    }
+
     // MARK: Private
 
     private static func expandGlobstar(pattern: String) -> [String] {