Improve performance of excluded files filter

.github/copilot-instructions.md

@@ -4,7 +4,7 @@ Linting rules are defined in `Source/SwiftLintBuiltInRules/Rules`. If someone me
 
 User-facing changes must be documented in the `CHANGELOG.md` file, which is organized by version. New entries always go into the "Main" section. They give credit to the person who has made the change and they reference the issue which has been fixed by the change.
 
-All changes on configuration options must be reflected in `Tests/IntegrationTests/default_rule_configurations.yml`. This can be achieved by running `swift run swiftlint-dev rules register`. Running this command is also necessary when new rules got added or removed to (un-)register them from/in the list of built-in rules and tests verifying all examples in rule descriptions.
+All changes on configuration options must be reflected in `Tests/IntegrationTests/Resources/default_rule_configurations.yml`. This can be achieved by running `swift run swiftlint-dev rules register`. Running this command is also necessary when new rules got added or removed to (un-)register them from/in the list of built-in rules and tests verifying all examples in rule descriptions.
 
 For some rules, there are dedicated tests in `Tests/BuiltInRulesTests`. However, they are typically not required as all the examples in the rule descriptions are automatically tested. The examples in the rule descriptions are also used to generate documentation for the rules. If an example presents a very pathological case, that's helpful for testing but not for user documentation, you can add the `excludeFromDocumentation: true` parameter to the example initializer. Important is that all examples in the rule description are verified by running `<RuleName>RuleGeneratedTests` for rule modified rules.
 

.swiftlint.yml

@@ -8,6 +8,7 @@ excluded:
   - assets
   - Tests/BuiltInRulesTests/Resources
   - Tests/FileSystemAccessTests/Resources
+  - Tests/IntegrationTests/Resources
 
 # Enabled/disabled rules
 analyzer_rules:

BUILD

@@ -70,6 +70,7 @@ swift_library(
     visibility = ["//visibility:public"],
     deps = [
         ":Yams.wrapper",
+        "@FilenameMatcher",
         "@SourceKittenFramework",
         "@SwiftSyntax//:SwiftIDEUtils_opt",
         "@SwiftSyntax//:SwiftOperators_opt",

CHANGELOG.md

@@ -31,6 +31,10 @@
 * Add new `unneeded_escaping` rule that detects closure parameters marked with
   `@escaping` that are never stored or captured escapingly.  
   [SimplyDanny](https://github.com/SimplyDanny)
+
+* Improve performance when exclude patterns resolve to a large set of files.  
+  [SimplyDanny](https://github.com/SimplyDanny)
+  [#5018](https://github.com/realm/SwiftLint/issues/5018)
 
 ### Bug Fixes
 

MODULE.bazel

@@ -15,6 +15,7 @@ bazel_dep(name = "rules_swift", version = "3.1.2", max_compatibility_level = 3,
 
 bazel_dep(name = "sourcekitten", version = "0.37.2", repo_name = "SourceKittenFramework")
 bazel_dep(name = "swift_argument_parser", version = "1.6.2", repo_name = "SwiftArgumentParser")
+bazel_dep(name = "swift-filename-matcher", version = "2.0.1", repo_name = "FilenameMatcher")
 bazel_dep(name = "yams", version = "6.2.0", repo_name = "Yams")
 
 swiftlint_repos = use_extension("//bazel:repos.bzl", "swiftlint_repos_bzlmod")

Package.swift

@@ -42,6 +42,7 @@ let package = Package(
         .package(url: "https://github.com/scottrhoyt/SwiftyTextTable.git", .upToNextMajor(from: "0.9.0")),
         .package(url: "https://github.com/JohnSundell/CollectionConcurrencyKit.git", .upToNextMajor(from: "0.2.0")),
         .package(url: "https://github.com/krzyzanowskim/CryptoSwift.git", .upToNextMajor(from: "1.9.0")),
+        .package(url: "https://github.com/ileitch/swift-filename-matcher", .upToNextMinor(from: "2.0.1")),
     ],
     targets: [
         .executableTarget(
@@ -65,6 +66,7 @@ let package = Package(
         .target(
             name: "SwiftLintFramework",
             dependencies: [
+                .product(name: "FilenameMatcher", package: "swift-filename-matcher"),
                 "SwiftLintBuiltInRules",
                 "SwiftLintCore",
                 "SwiftLintExtraRules",
@@ -96,6 +98,7 @@ let package = Package(
             dependencies: [
                 .product(name: "CryptoSwift", package: "CryptoSwift", condition: .when(platforms: [.linux])),
                 .target(name: "DyldWarningWorkaround", condition: .when(platforms: [.macOS])),
+                .product(name: "FilenameMatcher", package: "swift-filename-matcher"),
                 .product(name: "SourceKittenFramework", package: "SourceKitten"),
                 .product(name: "SwiftIDEUtils", package: "swift-syntax"),
                 .product(name: "SwiftOperators", package: "swift-syntax"),
@@ -167,6 +170,7 @@ let package = Package(
         .testTarget(
             name: "FileSystemAccessTests",
             dependencies: [
+                .product(name: "FilenameMatcher", package: "swift-filename-matcher"),
                 "SwiftLintFramework",
                 "TestHelpers",
                 "SwiftLintCoreMacros",
@@ -203,7 +207,7 @@ let package = Package(
                 "TestHelpers",
             ],
             exclude: [
-                "default_rule_configurations.yml"
+                "Resources",
             ],
             swiftSettings: swiftFeatures + targetedConcurrency // Set to strict once SwiftLintFramework is updated
         ),

Source/SwiftLintCore/Extensions/String+SwiftLint.swift

@@ -68,6 +68,9 @@ public extension String {
 
     /// Returns a new string, converting the path to a canonical absolute path.
     ///
+    /// > Important: This method might use an incorrect working directory internally. This can cause test failures
+    /// in Bazel builds but does not seem to cause trouble in production.
+    ///
     /// - returns: A new `String`.
     func absolutePathStandardized() -> String {
         bridge().absolutePathRepresentation().bridge().standardizingPath

Source/SwiftLintFramework/Configuration+CommandLine.swift

@@ -252,14 +252,8 @@ extension Configuration {
             guard options.forceExclude else {
                 return files
             }
-
             let scriptInputPaths = files.compactMap(\.path)
-
-            if options.useExcludingByPrefix {
-                return filterExcludedPathsByPrefix(in: scriptInputPaths)
-                    .map(SwiftLintFile.init(pathDeferringReading:))
-            }
-            return filterExcludedPaths(excludedPaths(), in: scriptInputPaths)
+            return filteredPaths(in: scriptInputPaths, excludeByPrefix: visitor.options.useExcludingByPrefix)
                 .map(SwiftLintFile.init(pathDeferringReading:))
         }
         if !options.quiet {
@@ -272,14 +266,12 @@ extension Configuration {
 
             queuedPrintError("\(options.capitalizedVerb) Swift files \(filesInfo)")
         }
-        let excludeLintableFilesBy = options.useExcludingByPrefix
-            ? Configuration.ExcludeBy.prefix
-            : .paths(excludedPaths: excludedPaths())
-        return options.paths.flatMap {
+        return visitor.options.paths.flatMap {
             self.lintableFiles(
                 inPath: $0,
-                forceExclude: options.forceExclude,
-                excludeBy: excludeLintableFilesBy)
+                forceExclude: visitor.options.forceExclude,
+                excludeByPrefix: visitor.options.useExcludingByPrefix
+            )
         }
     }
 

Source/SwiftLintFramework/Configuration/Configuration+LintableFiles.swift

@@ -1,25 +1,22 @@
+import FilenameMatcher
 import Foundation
 
 extension Configuration {
-    public enum ExcludeBy {
-        case prefix
-        case paths(excludedPaths: [String])
-    }
-
     // MARK: Lintable Paths
+
     /// Returns the files that can be linted by SwiftLint in the specified parent path.
     ///
     /// - parameter path:            The parent path in which to search for lintable files. Can be a directory or a
     ///                              file.
     /// - parameter forceExclude:    Whether or not excludes defined in this configuration should be applied even if
     ///                              `path` is an exact match.
-    /// - parameter excludeByPrefix: Whether or not uses excluding by prefix algorithm.
+    /// - parameter excludeByPrefix: Whether or not it uses the exclude-by-prefix algorithm.
     ///
     /// - returns: Files to lint.
     public func lintableFiles(inPath path: String,
                               forceExclude: Bool,
-                              excludeBy: ExcludeBy) -> [SwiftLintFile] {
-        lintablePaths(inPath: path, forceExclude: forceExclude, excludeBy: excludeBy)
+                              excludeByPrefix: Bool) -> [SwiftLintFile] {
+        lintablePaths(inPath: path, forceExclude: forceExclude, excludeByPrefix: excludeByPrefix)
             .parallelCompactMap {
                 SwiftLintFile(pathDeferringReading: $0)
             }
@@ -31,90 +28,65 @@ extension Configuration {
     ///                              file.
     /// - parameter forceExclude:    Whether or not excludes defined in this configuration should be applied even if
     ///                              `path` is an exact match.
-    /// - parameter excludeByPrefix: Whether or not uses excluding by prefix algorithm.
+    /// - parameter excludeByPrefix: Whether or not it uses the exclude-by-prefix algorithm.
     /// - parameter fileManager:     The lintable file manager to use to search for lintable files.
     ///
     /// - returns: Paths for files to lint.
-    internal func lintablePaths(
+    func lintablePaths(
         inPath path: String,
         forceExclude: Bool,
-        excludeBy: ExcludeBy,
+        excludeByPrefix: Bool,
         fileManager: some LintableFileManager = FileManager.default
     ) -> [String] {
+        let excluder = createExcluder(excludeByPrefix: excludeByPrefix)
+
+        // Handle single file path.
         if fileManager.isFile(atPath: path) {
-            if forceExclude {
-                switch excludeBy {
-                case .prefix:
-                    return filterExcludedPathsByPrefix(in: [path.absolutePathStandardized()])
-                case .paths(let excludedPaths):
-                    return filterExcludedPaths(excludedPaths, in: [path.absolutePathStandardized()])
-                }
-            }
-            // If path is a file and we're not forcing excludes, skip filtering with excluded/included paths
-            return [path]
+            return fileManager.filesToLint(
+                inPath: path,
+                rootDirectory: nil,
+                excluder: forceExclude ? excluder : .noExclusion
+            )
         }
 
-        let pathsForPath = includedPaths.isEmpty ? fileManager.filesToLint(inPath: path, rootDirectory: nil) : []
-        let includedPaths = self.includedPaths
-            .flatMap(Glob.resolveGlob)
-            .parallelFlatMap { fileManager.filesToLint(inPath: $0, rootDirectory: rootDirectory) }
-
-        switch excludeBy {
-        case .prefix:
-            return filterExcludedPathsByPrefix(in: pathsForPath, includedPaths)
-        case .paths(let excludedPaths):
-            return filterExcludedPaths(excludedPaths, in: pathsForPath, includedPaths)
+        // With no included paths, we lint everything in the given path.
+        if includedPaths.isEmpty {
+            return fileManager.filesToLint(
+                inPath: path,
+                rootDirectory: nil,
+                excluder: excluder
+            )
         }
-    }
-
-    /// Returns an array of file paths after removing the excluded paths as defined by this configuration.
-    ///
-    /// - parameter fileManager: The lintable file manager to use to expand the excluded paths into all matching paths.
-    /// - parameter paths:       The input paths to filter.
-    ///
-    /// - returns: The input paths after removing the excluded paths.
-    public func filterExcludedPaths(
-        _ excludedPaths: [String],
-        in paths: [String]...
-    ) -> [String] {
-        let allPaths = paths.flatMap(\.self)
-        #if os(Linux)
-        let result = NSMutableOrderedSet(capacity: allPaths.count)
-        result.addObjects(from: allPaths)
-        #else
-        let result = NSMutableOrderedSet(array: allPaths)
-        #endif
 
-        result.minusSet(Set(excludedPaths))
-        // swiftlint:disable:next force_cast
-        return result.map { $0 as! String }
+        // With included paths, we only lint those paths (after resolving globs).
+        return includedPaths
+            .flatMap(Glob.resolveGlob)
+            .parallelFlatMap {
+                fileManager.filesToLint(
+                    inPath: $0,
+                    rootDirectory: rootDirectory,
+                    excluder: excluder
+                )
+            }
     }
 
-    /// Returns the file paths that are excluded by this configuration using filtering by absolute path prefix.
-    ///
-    /// For cases when excluded directories contain many lintable files (e. g. Pods) it works faster than default
-    /// algorithm `filterExcludedPaths`.
-    ///
-    /// - returns: The input paths after removing the excluded paths.
-    public func filterExcludedPathsByPrefix(in paths: [String]...) -> [String] {
-        let allPaths = paths.flatMap(\.self)
-        let excludedPaths = self.excludedPaths
-            .parallelFlatMap { @Sendable in Glob.resolveGlob($0) }
-            .map { $0.absolutePathStandardized() }
-        return allPaths.filter { path in
-            !excludedPaths.contains { path.hasPrefix($0) }
-        }
+    func filteredPaths(in paths: [String], excludeByPrefix: Bool) -> [String] {
+        let excluder = createExcluder(excludeByPrefix: excludeByPrefix)
+        return paths.filter { !excluder.excludes(path: $0) }
     }
 
-    /// Returns the file paths that are excluded by this configuration after expanding them using the specified file
-    /// manager.
-    ///
-    /// - parameter fileManager: The file manager to get child paths in a given parent location.
-    ///
-    /// - returns: The expanded excluded file paths.
-    public func excludedPaths(fileManager: some LintableFileManager = FileManager.default) -> [String] {
-        excludedPaths
-            .flatMap(Glob.resolveGlob)
-            .parallelFlatMap { fileManager.filesToLint(inPath: $0, rootDirectory: rootDirectory) }
+    private func createExcluder(excludeByPrefix: Bool) -> Excluder {
+        if excludeByPrefix {
+            return .byPrefix(
+                prefixes: self.excludedPaths
+                    .flatMap { Glob.resolveGlob($0) }
+                    .map { $0.absolutePathStandardized() }
+              )
+        }
+        return .matching(
+            matchers: self.excludedPaths.flatMap {
+                Glob.createFilenameMatchers(root: rootDirectory, pattern: $0)
+            }
+        )
     }
 }

Source/SwiftLintFramework/Extensions/FileManager+SwiftLint.swift

@@ -1,17 +1,19 @@
+import FilenameMatcher
 import Foundation
 import SourceKittenFramework
 
 /// An interface for enumerating files that can be linted by SwiftLint.
 public protocol LintableFileManager {
     /// Returns all files that can be linted in the specified path. If the path is relative, it will be appended to the
-    /// specified root path, or currentt working directory if no root directory is specified.
+    /// specified root path, or current working directory if no root directory is specified.
     ///
     /// - parameter path:          The path in which lintable files should be found.
     /// - parameter rootDirectory: The parent directory for the specified path. If none is provided, the current working
     ///                            directory will be used.
+    /// - parameter excluder:     The excluder used to filter out files that should not be linted.
     ///
     /// - returns: Files to lint.
-    func filesToLint(inPath path: String, rootDirectory: String?) -> [String]
+    func filesToLint(inPath path: String, rootDirectory: String?, excluder: Excluder) -> [String]
 
     /// Returns the date when the file at the specified path was last modified. Returns `nil` if the file cannot be
     /// found or its last modification date cannot be determined.
@@ -29,22 +31,56 @@ public protocol LintableFileManager {
     func isFile(atPath path: String) -> Bool
 }
 
+/// An excluder for filtering out files that should not be linted.
+public enum Excluder {
+    /// Full matching excluder using filename matchers.
+    case matching(matchers: [FilenameMatcher])
+    /// Prefix-based excluder using path prefixes.
+    case byPrefix(prefixes: [String])
+    /// An excluder that does not exclude any files.
+    case noExclusion
+
+    func excludes(path: String) -> Bool {
+        switch self {
+        case let .matching(matchers):
+            matchers.contains(where: { $0.match(filename: path) })
+        case let .byPrefix(prefixes):
+            prefixes.contains(where: { path.hasPrefix($0) })
+        case .noExclusion:
+            false
+        }
+    }
+}
+
 extension FileManager: LintableFileManager {
-    public func filesToLint(inPath path: String, rootDirectory: String? = nil) -> [String] {
+    public func filesToLint(inPath path: String,
+                            rootDirectory: String? = nil,
+                            excluder: Excluder) -> [String] {
         let absolutePath = path.bridge()
             .absolutePathRepresentation(rootDirectory: rootDirectory ?? currentDirectoryPath).bridge()
             .standardizingPath
 
-        // if path is a file, it won't be returned in `enumerator(atPath:)`
+        // If path is a file, it won't be returned in `enumerator(atPath:)`.
         if absolutePath.bridge().isSwiftFile(), absolutePath.isFile {
-            return [absolutePath]
+            return excluder.excludes(path: absolutePath) ? [] : [absolutePath]
+        }
+
+        guard let enumerator = enumerator(atPath: absolutePath) else {
+            return []
         }
 
-        return subpaths(atPath: absolutePath)?.parallelCompactMap { element -> String? in
-            guard element.bridge().isSwiftFile() else { return nil }
+        var files = [String]()
+        while let element = enumerator.nextObject() as? String {
             let absoluteElementPath = absolutePath.bridge().appendingPathComponent(element)
-            return absoluteElementPath.isFile ? absoluteElementPath : nil
-        } ?? []
+            if absoluteElementPath.bridge().isSwiftFile(), absoluteElementPath.isFile {
+                if !excluder.excludes(path: absoluteElementPath) {
+                    files.append(absoluteElementPath)
+                }
+            } else if excluder.excludes(path: absoluteElementPath) {
+                enumerator.skipDescendants()
+            }
+        }
+        return files
     }
 
     public func modificationDate(forFileAtPath path: String) -> Date? {