Add simplified implementation of documentation input discovery (#1038)

* Add utility to file manager protocol to separate files and directories

* Update test file system to discover inputs using new input provider

* Generate inputs from symbol graphs when there is no catalog

* Use package access for new API that might change in a follow up PR

Author: d-ronnqvist

Sources/SwiftDocC/Infrastructure/DocumentationBundleFileTypes.swift

@@ -1,7 +1,7 @@
 /*
  This source file is part of the Swift.org open source project
 
- Copyright (c) 2021 Apple Inc. and the Swift project authors
+ Copyright (c) 2021-2024 Apple Inc. and the Swift project authors
  Licensed under Apache License v2.0 with Runtime Library Exception
 
  See https://swift.org/LICENSE.txt for license information
@@ -45,12 +45,12 @@ public enum DocumentationBundleFileTypes {
         return url.lastPathComponent.hasSuffix(symbolGraphFileExtension)
     }
 
-    private static let documentationBundleFileExtension = "docc"
-    /// Checks if a folder is a documentation bundle.
+    private static let documentationCatalogFileExtension = "docc"
+    /// Checks if a folder is a documentation catalog.
     /// - Parameter url: The folder to check.
-    /// - Returns: Whether or not the folder at `url` is a documentation bundle.
-    public static func isDocumentationBundle(_ url: URL) -> Bool {
-        return url.pathExtension.lowercased() == documentationBundleFileExtension
+    /// - Returns: Whether or not the folder at `url` is a documentation catalog.
+    public static func isDocumentationCatalog(_ url: URL) -> Bool {
+        url.pathExtension.lowercased() == documentationCatalogFileExtension
     }
 
     private static let infoPlistFileName = "Info.plist"
@@ -85,3 +85,10 @@ public enum DocumentationBundleFileTypes {
         return url.lastPathComponent == themeSettingsFileName
     }
 }
+
+extension DocumentationBundleFileTypes {
+    @available(*, deprecated, renamed: "isDocumentationCatalog(_:)", message: "Use 'isDocumentationCatalog(_:)' instead. This deprecated API will be removed after 6.1 is released")
+    public static func isDocumentationBundle(_ url: URL) -> Bool {
+        isDocumentationCatalog(url)
+    }
+}

Sources/SwiftDocC/Infrastructure/Input Discovery/DocumentationInputsProvider.swift

@@ -0,0 +1,254 @@
+/*
+ This source file is part of the Swift.org open source project
+
+ Copyright (c) 2024 Apple Inc. and the Swift project authors
+ Licensed under Apache License v2.0 with Runtime Library Exception
+
+ See https://swift.org/LICENSE.txt for license information
+ See https://swift.org/CONTRIBUTORS.txt for Swift project authors
+*/
+
+import Foundation
+import SymbolKit
+
+extension DocumentationContext {
+
+    /// A type that provides inputs for a unit of documentation.
+    package struct InputsProvider {
+        /// The file manager that the provider uses to read file and directory contents from the file system.
+        private var fileManager: FileManagerProtocol
+
+        /// Creates a new documentation inputs provider.
+        /// - Parameter fileManager: The file manager that the provider uses to read file and directory contents from the file system.
+        package init(fileManager: FileManagerProtocol) {
+            self.fileManager = fileManager
+        }
+
+        /// Creates a new documentation inputs provider.
+        package init() {
+            self.init(fileManager: FileManager.default)
+        }
+    }
+}
+
+// MARK: Catalog discovery
+
+extension DocumentationContext.InputsProvider {
+
+    private typealias FileTypes = DocumentationBundleFileTypes
+
+    /// A discovered documentation catalog.
+    package struct CatalogURL {
+        let url: URL
+    }
+
+    struct MultipleCatalogsError: DescribedError {
+        let startingPoint: URL
+        let catalogs: [URL]
+
+        var errorDescription: String {
+            """
+            Found multiple documentation catalogs in \(startingPoint.standardizedFileURL.path):
+            \(catalogs.map { ($0.relative(to: startingPoint) ?? $0).standardizedFileURL.path }.sorted().map { " - \($0)" }.joined(separator: "\n"))
+            """
+        }
+    }
+
+    /// Traverses the file system from the given starting point to find a documentation catalog.
+    /// - Parameters:
+    ///   - startingPoint: The top of the directory hierarchy that the provider traverses to find a documentation catalog.
+    ///   - allowArbitraryCatalogDirectories: Whether to treat the starting point as a documentation catalog if the provider doesn't find an actual catalog on the file system.
+    /// - Returns: The found documentation catalog.
+    /// - Throws: If the directory hierarchy contains more than one documentation catalog.
+    package func findCatalog(
+        startingPoint: URL,
+        allowArbitraryCatalogDirectories: Bool = false
+    ) throws -> CatalogURL? {
+        var foundCatalogs: [URL] = []
+
+        var urlsToCheck = [startingPoint]
+        while !urlsToCheck.isEmpty {
+            let url = urlsToCheck.removeFirst()
+
+            guard !FileTypes.isDocumentationCatalog(url) else {
+                // Don't look for catalogs inside of other catalogs.
+                foundCatalogs.append(url)
+                continue
+            }
+
+            urlsToCheck.append(contentsOf: try fileManager.contentsOfDirectory(at: url, options: .skipsHiddenFiles).directories)
+        }
+
+        guard foundCatalogs.count <= 1 else {
+            throw MultipleCatalogsError(startingPoint: startingPoint, catalogs: foundCatalogs)
+        }
+
+        let catalogURL = foundCatalogs.first
+        // If the provider didn't find a catalog, check if the root should be treated as a catalog
+        ?? (allowArbitraryCatalogDirectories ? startingPoint : nil)
+
+        return catalogURL.map(CatalogURL.init)
+    }
+}
+
+// MARK: Inputs creation
+
+extension DocumentationContext {
+    package typealias Inputs = DocumentationBundle
+}
+
+extension DocumentationContext.InputsProvider {
+
+    package typealias Options = BundleDiscoveryOptions
+
+    /// Creates a collection of documentation inputs from the content of the given documentation catalog.
+    /// 
+    /// - Parameters:
+    ///   - catalogURL: The location of a discovered documentation catalog.
+    ///   - options: Options to configure how the provider creates the documentation inputs.
+    /// - Returns: Inputs that categorize the files of the given catalog.
+    package func makeInputs(contentOf catalogURL: CatalogURL, options: Options) throws -> DocumentationContext.Inputs {
+        let url = catalogURL.url
+        let shallowContent = try fileManager.contentsOfDirectory(at: url, options: [.skipsHiddenFiles]).files
+        let infoPlistData = try shallowContent
+            .first(where: FileTypes.isInfoPlistFile)
+            .map { try fileManager.contents(of: $0) }
+
+        let info = try DocumentationContext.Inputs.Info(
+            from: infoPlistData,
+            bundleDiscoveryOptions: options,
+            derivedDisplayName: url.deletingPathExtension().lastPathComponent
+        )
+
+        let foundContents = try findContents(in: url)
+        return DocumentationContext.Inputs(
+            info: info,
+            symbolGraphURLs:  foundContents.symbolGraphs + options.additionalSymbolGraphFiles,
+            markupURLs:       foundContents.markup,
+            miscResourceURLs: foundContents.resources,
+            customHeader:  shallowContent.first(where: FileTypes.isCustomHeader),
+            customFooter:  shallowContent.first(where: FileTypes.isCustomFooter),
+            themeSettings: shallowContent.first(where: FileTypes.isThemeSettingsFile)
+        )
+    }
+
+    /// Finds all the markup files, resource files, and symbol graph files in the given directory.
+    private func findContents(in startURL: URL) throws -> (markup: [URL], resources: [URL], symbolGraphs: [URL]) {
+        // Find all the files
+        var foundMarkup:       [URL] = []
+        var foundResources:    [URL] = []
+        var foundSymbolGraphs: [URL] = []
+
+        var urlsToCheck = [startURL]
+        while !urlsToCheck.isEmpty {
+            let url = urlsToCheck.removeFirst()
+
+            var (files, directories) = try fileManager.contentsOfDirectory(at: url, options: .skipsHiddenFiles)
+
+            urlsToCheck.append(contentsOf: directories)
+
+            // Group the found files by type
+            let markupPartitionIndex = files.partition(by: FileTypes.isMarkupFile)
+            var nonMarkupFiles = files[..<markupPartitionIndex]
+            let symbolGraphPartitionIndex = nonMarkupFiles.partition(by: FileTypes.isSymbolGraphFile)
+
+            foundMarkup.append(contentsOf:       files[markupPartitionIndex...]               )
+            foundResources.append(contentsOf:    nonMarkupFiles[..<symbolGraphPartitionIndex] )
+            foundSymbolGraphs.append(contentsOf: nonMarkupFiles[symbolGraphPartitionIndex...] )
+        }
+
+        return (markup: foundMarkup, resources: foundResources, symbolGraphs: foundSymbolGraphs)
+    }
+}
+
+// MARK: Create without catalog
+
+extension DocumentationContext.InputsProvider {
+    /// Creates a collection of documentation inputs from the symbol graph files and other command line options.
+    ///
+    /// - Parameter options: Options to configure how the provider creates the documentation inputs.
+    /// - Returns: Inputs that categorize the files of the given catalog.
+    package func makeInputsFromSymbolGraphs(options: Options) throws -> DocumentationContext.Inputs? {
+        guard !options.additionalSymbolGraphFiles.isEmpty else {
+            return nil
+        }
+
+        // Find all the unique module names from the symbol graph files and generate a top level module page for each of them.
+        var moduleNames = Set<String>()
+        for url in options.additionalSymbolGraphFiles {
+            let data = try fileManager.contents(of: url)
+            let container = try JSONDecoder().decode(SymbolGraphModuleContainer.self, from: data)
+            moduleNames.insert(container.module.name)
+        }
+        let derivedDisplayName = moduleNames.count == 1 ? moduleNames.first : nil
+
+        let info = try DocumentationContext.Inputs.Info(bundleDiscoveryOptions: options, derivedDisplayName: derivedDisplayName)
+
+        var topLevelPages: [URL] = []
+        if moduleNames.count == 1, let moduleName = moduleNames.first, moduleName != info.displayName {
+            let tempURL = fileManager.uniqueTemporaryDirectory()
+            try? fileManager.createDirectory(at: tempURL, withIntermediateDirectories: true, attributes: nil)
+
+            let url = tempURL.appendingPathComponent("\(moduleName).md")
+            topLevelPages.append(url)
+            try fileManager.createFile(
+                at: url,
+                contents: Data("""
+                # ``\(moduleName)``
+                
+                @Metadata {
+                  @DisplayName("\(info.displayName)")
+                }
+                """.utf8),
+                options: .atomic
+            )
+        }
+
+        return DocumentationBundle(
+            info: info,
+            symbolGraphURLs: options.additionalSymbolGraphFiles,
+            markupURLs: topLevelPages,
+            miscResourceURLs: []
+        )
+    }
+}
+
+/// A wrapper type that decodes only the module in the symbol graph.
+private struct SymbolGraphModuleContainer: Decodable {
+    /// The decoded symbol graph module.
+    let module: SymbolGraph.Module
+
+    typealias CodingKeys = SymbolGraph.CodingKeys
+
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: CodingKeys.self)
+
+        self.module = try container.decode(SymbolGraph.Module.self, forKey: .module)
+    }
+}
+
+// MARK: Discover and create
+
+extension DocumentationContext.InputsProvider {
+    /// Traverses the file system from the given starting point to find a documentation catalog and creates a collection of documentation inputs from that catalog.
+    ///
+    /// If the provider can't find a catalog, it will try to create documentation inputs from the option's symbol graph files.
+    ///
+    /// - Parameters:
+    ///   - startingPoint: The top of the directory hierarchy that the provider traverses to find a documentation catalog.
+    ///   - allowArbitraryCatalogDirectories: Whether to treat the starting point as a documentation catalog if the provider doesn't find an actual catalog on the file system.
+    ///   - options: Options to configure how the provider creates the documentation inputs.
+    /// - Returns: The documentation inputs for the found documentation catalog, or `nil` if the directory hierarchy doesn't contain a catalog.
+    /// - Throws: If the directory hierarchy contains more than one documentation catalog.
+    package func inputs(
+        startingPoint: URL,
+        allowArbitraryCatalogDirectories: Bool = false,
+        options: Options
+    ) throws -> DocumentationContext.Inputs? {
+        if let catalogURL = try findCatalog(startingPoint: startingPoint, allowArbitraryCatalogDirectories: allowArbitraryCatalogDirectories) {
+            try makeInputs(contentOf: catalogURL, options: options)
+        } else {
+            try makeInputsFromSymbolGraphs(options: options)
+        }
+    }
+}

Sources/SwiftDocC/Infrastructure/Workspace/LocalFileSystemDataProvider+BundleDiscovery.swift

@@ -40,7 +40,7 @@ extension LocalFileSystemDataProvider {
             preconditionFailure("Expected directory object at path '\(root.url.absoluteString)'.")
         }
 
-        if DocumentationBundleFileTypes.isDocumentationBundle(rootDirectory.url) {
+        if DocumentationBundleFileTypes.isDocumentationCatalog(rootDirectory.url) {
             bundles.append(try createBundle(rootDirectory, rootDirectory.children, options: options))
         } else {
             // Recursively descend when the current root directory isn't a documentation bundle.
@@ -125,7 +125,7 @@ extension LocalFileSystemDataProvider {
     ///   - recursive: If `true`, this function will recursively check the files of all directories in the array. If `false`, it will ignore all directories.
     /// - Returns: A list of all the non-markup files.
     private func findNonMarkupFiles(_ bundleChildren: [FSNode], recursive: Bool) -> [FSNode.File] {
-        return bundleChildren.files(recursive: recursive) { !DocumentationBundleFileTypes.isMarkupFile($0.url) }
+        bundleChildren.files(recursive: recursive) { !DocumentationBundleFileTypes.isMarkupFile($0.url) && !DocumentationBundleFileTypes.isSymbolGraphFile($0.url) }
     }
 
     private func findCustomHeader(_ bundleChildren: [FSNode]) -> FSNode.File? {

Sources/SwiftDocC/Utility/FileManagerProtocol.swift

@@ -87,6 +87,14 @@ package protocol FileManagerProtocol {
     ///   - options: Options for writing the data. Provide `nil` to use the default
     ///              writing options of the file manager.
     func createFile(at location: URL, contents: Data, options writingOptions: NSData.WritingOptions?) throws
+
+    /// Performs a shallow search of the specified directory and returns the file and directory URLs for the contained items.
+    ///
+    /// - Parameters:
+    ///   - url: The URL for the directory whose contents to enumerate.
+    ///   - mark: Options for the enumeration. Because this method performs only shallow enumerations, the only supported option is `skipsHiddenFiles`.
+    /// - Returns: The URLs of each file and directory that's contained in `url`.
+    func contentsOfDirectory(at url: URL, options mask: FileManager.DirectoryEnumerationOptions) throws -> (files: [URL], directories: [URL])
 }
 
 extension FileManagerProtocol {
@@ -123,4 +131,17 @@ extension FileManager: FileManagerProtocol {
     package func uniqueTemporaryDirectory() -> URL {
         temporaryDirectory.appendingPathComponent(ProcessInfo.processInfo.globallyUniqueString, isDirectory: true)
     }
+
+    // This method doesn't exist on `FileManger`. We define it on the protocol to enable the FileManager to provide an implementation that avoids repeated reads to discover which contained items are files and which are directories.
+    package func contentsOfDirectory(at url: URL, options mask: DirectoryEnumerationOptions) throws -> (files: [URL], directories: [URL]) {
+        var allContents = try contentsOfDirectory(at: url, includingPropertiesForKeys: [.isDirectoryKey], options: .skipsHiddenFiles)
+
+        let partitionIndex = try allContents.partition {
+            try $0.resourceValues(forKeys: [.isDirectoryKey]).isDirectory == true
+        }
+        return (
+            files:       Array( allContents[..<partitionIndex] ),
+            directories: Array( allContents[partitionIndex...] )
+        )
+    }
 }