Add support for standalone articles and tutorials

Adds support for ConvertService to generate documentation for single
articles without a top-level root page and for uncurated tutorials.

rdar://105374582

Author: franklinsch

Sources/SwiftDocC/DocumentationService/Convert/ConvertService+DataProvider.swift

@@ -24,10 +24,23 @@ extension ConvertService {
             info: DocumentationBundle.Info,
             symbolGraphs: [Data],
             markupFiles: [Data],
+            tutorialFiles: [Data],
             miscResourceURLs: [URL]
         ) {
-            let symbolGraphURLs = symbolGraphs.map { registerFile(contents: $0, isMarkupFile: false) }
-            let markupFileURLs = markupFiles.map { registerFile(contents: $0, isMarkupFile: true) }
+            let symbolGraphURLs = symbolGraphs.map { registerFile(contents: $0, pathExtension: nil) }
+            let markupFileURLs = markupFiles.map { markupFile in
+                registerFile(
+                    contents: markupFile,
+                    pathExtension:
+                        DocumentationBundleFileTypes.referenceFileExtension
+                )
+            } + tutorialFiles.map { tutorialFile in
+                registerFile(
+                    contents: tutorialFile,
+                    pathExtension:
+                        DocumentationBundleFileTypes.tutorialFileExtension
+                )
+            }
 
             bundles.append(
                 DocumentationBundle(
@@ -39,8 +52,8 @@ extension ConvertService {
             )
         }
 
-        private mutating func registerFile(contents: Data, isMarkupFile: Bool) -> URL {
-            let url = Self.createURL(isMarkupFile: isMarkupFile)
+        private mutating func registerFile(contents: Data, pathExtension: String?) -> URL {
+            let url = Self.createURL(pathExtension: pathExtension)
             files[url] = contents
             return url
         }
@@ -50,11 +63,11 @@ extension ConvertService {
         /// The URL this function generates for a resource is not derived from the resource itself, because it doesn't need to be. The
         /// ``DocumentationWorkspaceDataProvider`` model revolves around retrieving resources by their URL. In our use
         /// case, our resources are not file URLs so we generate a URL for each resource.
-        static private func createURL(isMarkupFile: Bool) -> URL {
+        static private func createURL(pathExtension: String? = nil) -> URL {
             var url = URL(string: "docc-service:/\(UUID().uuidString)")!
 
-            if isMarkupFile {
-                url.appendPathExtension(DocumentationBundleFileTypes.referenceFileExtension)
+            if let pathExtension = pathExtension {
+                url.appendPathExtension(pathExtension)
             }
 
             return url

Sources/SwiftDocC/DocumentationService/Convert/ConvertService.swift

@@ -136,6 +136,7 @@ public struct ConvertService: DocumentationService {
                     info: request.bundleInfo,
                     symbolGraphs: request.symbolGraphs,
                     markupFiles: request.markupFiles,
+                    tutorialFiles: request.tutorialFiles,
                     miscResourceURLs: request.miscResourceURLs
                 )
 
@@ -145,6 +146,10 @@ public struct ConvertService: DocumentationService {
             let context = try DocumentationContext(dataProvider: workspace)
             context.knownDisambiguatedSymbolPathComponents = request.knownDisambiguatedSymbolPathComponents
 
+            // Enable support for generating documentation for standalone articles and tutorials.
+            context.allowsRegisteringArticlesWithoutTechnologyRoot = true
+            context.allowsRegisteringUncuratedTutorials = true
+            
             if let linkResolvingServer = linkResolvingServer {
                 let resolver = try OutOfProcessReferenceResolver(
                     bundleIdentifier: request.bundleInfo.identifier,

Sources/SwiftDocC/DocumentationService/Models/Services/Convert/ConvertRequest.swift

@@ -105,12 +105,16 @@ public struct ConvertRequest: Codable {
     /// - ``DocumentationBundle/symbolGraphURLs``
     public var symbolGraphs: [Data]
 
-    /// The markup file data included in the documentation bundle to convert.
+    /// The article and documentation extension file data included in the documentation bundle to convert.
     ///
     /// ## See Also
     /// - ``DocumentationBundle/markupURLs``
     public var markupFiles: [Data]
 
+    
+    /// The tutorial file data included in the documentation bundle to convert.
+    public var tutorialFiles: [Data]
+    
     /// The on-disk resources in the documentation bundle to convert.
     ///
     /// ## See Also
@@ -153,6 +157,7 @@ public struct ConvertRequest: Codable {
         self.symbolGraphs = symbolGraphs
         self.knownDisambiguatedSymbolPathComponents = knownDisambiguatedSymbolPathComponents
         self.markupFiles = markupFiles
+        self.tutorialFiles = []
         self.miscResourceURLs = miscResourceURLs
         self.featureFlags = FeatureFlags()
 
@@ -174,7 +179,8 @@ public struct ConvertRequest: Codable {
     ///   - symbolGraphs: The symbols graph data included in the documentation bundle to convert.
     ///   - knownDisambiguatedSymbolPathComponents: The mapping of external symbol identifiers to
     ///   known disambiguated symbol path components.
-    ///   - markupFiles: The markup file data included in the documentation bundle to convert.
+    ///   - markupFiles: The article and documentation extension file data included in the documentation bundle to convert.
+    ///   - tutorialFiles: The tutorial file data included in the documentation bundle to convert.
     ///   - miscResourceURLs: The on-disk resources in the documentation bundle to convert.
     public init(
         bundleInfo: DocumentationBundle.Info,
@@ -186,6 +192,7 @@ public struct ConvertRequest: Codable {
         symbolGraphs: [Data],
         knownDisambiguatedSymbolPathComponents: [String: [String]]? = nil,
         markupFiles: [Data],
+        tutorialFiles: [Data] = [],
         miscResourceURLs: [URL]
     ) {
         self.externalIDsToConvert = externalIDsToConvert
@@ -195,6 +202,7 @@ public struct ConvertRequest: Codable {
         self.symbolGraphs = symbolGraphs
         self.knownDisambiguatedSymbolPathComponents = knownDisambiguatedSymbolPathComponents
         self.markupFiles = markupFiles
+        self.tutorialFiles = tutorialFiles
         self.miscResourceURLs = miscResourceURLs
         self.bundleInfo = bundleInfo
         self.featureFlags = featureFlags

Sources/SwiftDocC/Infrastructure/DocumentationBundleFileTypes.swift

@@ -21,7 +21,7 @@ public enum DocumentationBundleFileTypes {
         return url.pathExtension.lowercased() == referenceFileExtension
     }
 
-    private static let tutorialFileExtension = "tutorial"
+    static let tutorialFileExtension = "tutorial"
     /// Checks if a file is a tutorial file.
     /// - Parameter url: The file to check.
     /// - Returns: Whether or not the file at `url` is a tutorial file.

Sources/SwiftDocC/Infrastructure/DocumentationContext.swift

@@ -139,6 +139,19 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
             }
         }
     }
+    
+    /// Controls whether bundle registration should allow registering articles when no technology root is defined.
+    ///
+    /// Set this property to `true` to enable registering documentation for standalone articles,
+    /// for example when using ``ConvertService``.
+    var allowsRegisteringArticlesWithoutTechnologyRoot: Bool = false
+    
+    /// Controls whether tutorials that aren't curated in a tutorials overview page are registered and translated.
+    ///
+    /// Set this property to `true` to enable registering documentation for standalone tutorials,
+    /// for example when ``ConvertService``.
+    var allowsRegisteringUncuratedTutorials: Bool = false
+    
     /// The set of all manually curated references if `shouldStoreManuallyCuratedReferences` was true at the time of processing and has remained `true` since.. Nil if curation has not been processed yet.
     public private(set) var manuallyCuratedReferences: Set<ResolvedTopicReference>?
 
@@ -2133,7 +2146,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
 
         // Articles that will be automatically curated can be resolved but they need to be pre registered before resolving links.
         let rootNodeForAutomaticCuration = soleRootModuleReference.flatMap(topicGraph.nodeWithReference(_:))
-        if rootNodeForAutomaticCuration != nil {
+        if allowsRegisteringArticlesWithoutTechnologyRoot || rootNodeForAutomaticCuration != nil {
             otherArticles = registerArticles(otherArticles, in: bundle)
             try shouldContinueRegistration()
         }

Sources/SwiftDocC/Model/Rendering/RenderNodeTranslator.swift

@@ -45,6 +45,9 @@ public struct RenderNodeTranslator: SemanticVisitor {
     /// Whether the documentation converter should include access level information for symbols.
     var shouldEmitSymbolAccessLevels: Bool
 
+    /// Whether tutorials that are not curated in a tutorials overview should be translated.
+    var shouldRenderUncuratedTutorials: Bool = false
+    
     /// The source repository where the documentation's sources are hosted.
     var sourceRepository: SourceRepository?
 
@@ -116,26 +119,25 @@ public struct RenderNodeTranslator: SemanticVisitor {
         var node = RenderNode(identifier: identifier, kind: .tutorial)
 
         var hierarchyTranslator = RenderHierarchyTranslator(context: context, bundle: bundle)
-        guard let hierarchy = hierarchyTranslator.visitTechnologyNode(identifier) else {
+        
+        if let hierarchy = hierarchyTranslator.visitTechnologyNode(identifier) {
+            let technology = try! context.entity(with: hierarchy.technology).semantic as! Technology
+            node.hierarchy = hierarchy.hierarchy
+            node.metadata.category = technology.name
+            node.metadata.categoryPathComponent = hierarchy.technology.url.lastPathComponent
+        } else if !context.allowsRegisteringArticlesWithoutTechnologyRoot {
             // This tutorial is not curated, so we don't generate a render node.
             // We've warned about this during semantic analysis.
             return nil
         }
 
-        let technology = try! context.entity(with: hierarchy.technology).semantic as! Technology
-        
         node.metadata.title = tutorial.intro.title
         node.metadata.role = contentRenderer.role(for: .tutorial).rawValue
 
         collectedTopicReferences.append(contentsOf: hierarchyTranslator.collectedTopicReferences)
 
-        node.hierarchy = hierarchy.hierarchy
-        node.metadata.category = technology.name
-        
         let documentationNode = try! context.entity(with: identifier)
         node.variants = variants(for: documentationNode)
-        
-        node.metadata.categoryPathComponent = hierarchy.technology.url.lastPathComponent
 
         var intro = visitIntro(tutorial.intro) as! IntroRenderSection
         intro.estimatedTimeInMinutes = tutorial.durationMinutes

Tests/SwiftDocCTests/DocumentationService/ConvertService/ConvertServiceTests.swift

@@ -524,6 +524,145 @@ class ConvertServiceTests: XCTestCase {
         }
     }
 
+    func testConvertSingleArticlePage() throws {
+        let articleFile = Bundle.module.url(
+            forResource: "StandaloneArticle",
+            withExtension: "md",
+            subdirectory: "Test Resources"
+        )!
+        
+        let article = try Data(contentsOf: articleFile)
+        
+        let request = ConvertRequest(
+            bundleInfo: testBundleInfo,
+            externalIDsToConvert: nil,
+            documentPathsToConvert: nil,
+            symbolGraphs: [],
+            knownDisambiguatedSymbolPathComponents: nil,
+            markupFiles: [article],
+            miscResourceURLs: []
+        )
+        
+        try processAndAssert(request: request) { message in
+            XCTAssertEqual(message.type, "convert-response")
+            XCTAssertEqual(message.identifier, "test-identifier-response")
+            
+            let response = try JSONDecoder().decode(
+                ConvertResponse.self, from: XCTUnwrap(message.payload)
+            )
+            
+            XCTAssertEqual(response.renderNodes.count, 1)
+            let data = try XCTUnwrap(response.renderNodes.first)
+            let renderNode = try JSONDecoder().decode(RenderNode.self, from: data)
+            
+            XCTAssertEqual(
+                renderNode.metadata.externalID,
+                nil
+            )
+            
+            XCTAssertEqual(renderNode.kind, .article)
+            
+            XCTAssertEqual(renderNode.abstract?.count, 1)
+            
+            XCTAssertEqual(
+                renderNode.abstract?.first,
+                .text("An article abstract.")
+            )
+        }
+    }
+    
+    func testConvertSingleTutorial() throws {
+        let tutorialFile = Bundle.module.url(
+            forResource: "StandaloneTutorial",
+            withExtension: "tutorial",
+            subdirectory: "Test Resources"
+        )!
+        
+        let tutorial = try Data(contentsOf: tutorialFile)
+        
+        let request = ConvertRequest(
+            bundleInfo: testBundleInfo,
+            externalIDsToConvert: nil,
+            documentPathsToConvert: nil,
+            symbolGraphs: [],
+            knownDisambiguatedSymbolPathComponents: nil,
+            markupFiles: [],
+            tutorialFiles: [tutorial],
+            miscResourceURLs: []
+        )
+        
+        try processAndAssert(request: request) { message in
+            XCTAssertEqual(message.type, "convert-response")
+            XCTAssertEqual(message.identifier, "test-identifier-response")
+            
+            let response = try JSONDecoder().decode(
+                ConvertResponse.self, from: XCTUnwrap(message.payload)
+            )
+            
+            XCTAssertEqual(response.renderNodes.count, 1)
+            let data = try XCTUnwrap(response.renderNodes.first)
+            let renderNode = try JSONDecoder().decode(RenderNode.self, from: data)
+            
+            XCTAssertEqual(
+                renderNode.metadata.externalID,
+                nil
+            )
+            
+            XCTAssertEqual(renderNode.kind, .tutorial)
+            
+            XCTAssertEqual(
+                renderNode.metadata.title,
+                "Standalone Tutorial"
+            )
+        }
+    }
+    
+    func testConvertSingleTutorialOverview() throws {
+        let tutorialOverviewFile = Bundle.module.url(
+            forResource: "StandaloneTutorialOverview",
+            withExtension: "tutorial",
+            subdirectory: "Test Resources"
+        )!
+        
+        let tutorialOverview = try Data(contentsOf: tutorialOverviewFile)
+        
+        let request = ConvertRequest(
+            bundleInfo: testBundleInfo,
+            externalIDsToConvert: nil,
+            documentPathsToConvert: nil,
+            symbolGraphs: [],
+            knownDisambiguatedSymbolPathComponents: nil,
+            markupFiles: [],
+            tutorialFiles: [tutorialOverview],
+            miscResourceURLs: []
+        )
+        
+        try processAndAssert(request: request) { message in
+            XCTAssertEqual(message.type, "convert-response")
+            XCTAssertEqual(message.identifier, "test-identifier-response")
+            
+            let response = try JSONDecoder().decode(
+                ConvertResponse.self, from: XCTUnwrap(message.payload)
+            )
+            
+            XCTAssertEqual(response.renderNodes.count, 1)
+            let data = try XCTUnwrap(response.renderNodes.first)
+            let renderNode = try JSONDecoder().decode(RenderNode.self, from: data)
+            
+            XCTAssertEqual(
+                renderNode.metadata.externalID,
+                nil
+            )
+            
+            XCTAssertEqual(renderNode.kind, .overview)
+            
+            XCTAssertEqual(
+                renderNode.metadata.title,
+                "Standalone Tutorial Overview"
+            )
+        }
+    }
+    
     func processAndAssertResponseContents(
         expectedRenderNodePaths: [String],
         includesRenderReferenceStore: Bool,

Tests/SwiftDocCTests/Test Resources/StandaloneArticle.md

@@ -0,0 +1,7 @@
+# My Article
+
+An article abstract.
+
+This is an overview of the article.
+
+<!-- Copyright (c) 2023 Apple Inc and the Swift Project authors. All Rights Reserved. -->

Tests/SwiftDocCTests/Test Resources/StandaloneTutorial.tutorial

@@ -0,0 +1,8 @@
+@Tutorial {
+   @Intro(title: "Standalone Tutorial") {
+
+      This is the tutorial abstract.
+   }
+}
+
+<!-- Copyright (c) 2023 Apple Inc and the Swift Project authors. All Rights Reserved. -->

Tests/SwiftDocCTests/Test Resources/StandaloneTutorialOverview.tutorial

@@ -0,0 +1,6 @@
+@Tutorials(name: "Standalone Tutorial Overview") {
+   @Intro(title: "Standalone Tutorial Overview") {
+   }
+}
+
+<!-- Copyright (c) 2023 Apple Inc and the Swift Project authors. All Rights Reserved. -->