Serialize linkable entities incrementally for performance

Author: d-ronnqvist

Sources/SwiftDocC/Infrastructure/ConvertActionConverter.swift

@@ -77,7 +77,6 @@ package enum ConvertActionConverter {
 
         // Arrays to gather additional metadata if `emitDigest` is `true`.
         var indexingRecords = [IndexingRecord]()
-        var linkSummaries = [LinkDestinationSummary]()
         var assets = [RenderReferenceType : [any RenderReference]]()
         var coverageInfo = [CoverageDataEntry]()
         let coverageFilterClosure = documentationCoverageOptions.generateFilterClosure()
@@ -151,16 +150,18 @@ package enum ConvertActionConverter {
                         let nodeLinkSummaries = entity.externallyLinkableElementSummaries(context: context, renderNode: renderNode, includeTaskGroups: true)
                         let nodeIndexingRecords = try renderNode.indexingRecords(onPage: identifier)
 
+                        for linkSummary in nodeLinkSummaries {
+                            try outputConsumer.consumeIncremental(linkableElementSummary: linkSummary)
+                        }
                         resultsGroup.async(queue: resultsSyncQueue) {
                             assets.merge(renderNode.assetReferences, uniquingKeysWith: +)
-                            linkSummaries.append(contentsOf: nodeLinkSummaries)
                             indexingRecords.append(contentsOf: nodeIndexingRecords)
                         }
                     } else if FeatureFlags.current.isLinkHierarchySerializationEnabled {
                         let nodeLinkSummaries = entity.externallyLinkableElementSummaries(context: context, renderNode: renderNode, includeTaskGroups: false)
 
-                        resultsGroup.async(queue: resultsSyncQueue) {
-                            linkSummaries.append(contentsOf: nodeLinkSummaries)
+                        for linkSummary in nodeLinkSummaries {
+                            try outputConsumer.consumeIncremental(linkableElementSummary: linkSummary)
                         }
                     }
                 } catch {
@@ -180,7 +181,7 @@ package enum ConvertActionConverter {
         if emitDigest {
             signposter.withIntervalSignpost("Emit digest", id: signposter.makeSignpostID()) {
                 do {
-                    try outputConsumer.consume(linkableElementSummaries: linkSummaries)
+                    try outputConsumer.finishedConsumingLinkElementSummaries()
                     try outputConsumer.consume(indexingRecords: indexingRecords)
                     try outputConsumer.consume(assets: assets)
                 } catch {
@@ -196,7 +197,7 @@ package enum ConvertActionConverter {
                     try outputConsumer.consume(linkResolutionInformation: serializableLinkInformation)
 
                     if !emitDigest {
-                        try outputConsumer.consume(linkableElementSummaries: linkSummaries)
+                        try outputConsumer.finishedConsumingLinkElementSummaries()
                     }
                 } catch {
                     recordProblem(from: error, in: &conversionProblems, withIdentifier: "link-resolver")

Sources/SwiftDocC/Infrastructure/ConvertOutputConsumer.swift

@@ -27,6 +27,12 @@ public protocol ConvertOutputConsumer {
     func consume(assetsInBundle bundle: DocumentationBundle) throws
 
     /// Consumes the linkable element summaries produced during a conversion.
+    /// > Warning: This method might be called concurrently.
+    func consumeIncremental(linkableElementSummary: LinkDestinationSummary) throws
+    /// Consumes the linkable element summaries produced during a conversion.
+    func finishedConsumingLinkElementSummaries() throws
+    
+    @available(*, deprecated, renamed: "consume(linkableElementSummary:)", message: "Use 'consume(linkableElementSummary:)' instead. This deprecated API will be removed after 6.3 is released")
     func consume(linkableElementSummaries: [LinkDestinationSummary]) throws
 
     /// Consumes the indexing records produced during a conversion.
@@ -60,8 +66,15 @@ public extension ConvertOutputConsumer {
     func consume(linkResolutionInformation: SerializableLinkResolutionInformation) throws {}
 }
 
+// Default implementations to avoid a source breaking change from introducing new protocol requirements
+public extension ConvertOutputConsumer {
+    func consumeIncremental(linkableElementSummary: LinkDestinationSummary) throws {}
+    func finishedConsumingLinkElementSummaries() throws {}
+}
+
 // Default implementation so that conforming types don't need to implement deprecated API.
 public extension ConvertOutputConsumer {
+    func consume(linkableElementSummaries: [LinkDestinationSummary]) throws {}
     func _deprecated_consume(problems: [Problem]) throws {}
 }
 

Sources/SwiftDocC/Utility/FeatureFlags.swift

@@ -20,7 +20,7 @@ public struct FeatureFlags: Codable {
     /// Whether or not support for emitting a serialized version of the local link resolution information is enabled.
     public var isLinkHierarchySerializationEnabled = true
 
-    @available(*, deprecated, renamed: "isLinkHierarchySerializationEnabled", message: "Use 'isLinkHierarchySerializationEnabled' instead. This deprecated API will be removed after 6.2 is released")
+    @available(*, deprecated, renamed: "isLinkHierarchySerializationEnabled", message: "Use 'isLinkHierarchySerializationEnabled' instead. This deprecated API will be removed after 6.3 is released")
     public var isExperimentalLinkHierarchySerializationEnabled: Bool {
         get { isLinkHierarchySerializationEnabled }
         set { isLinkHierarchySerializationEnabled = newValue }

Sources/SwiftDocCUtilities/Action/Actions/Convert/ConvertFileWritingConsumer.swift

@@ -152,6 +152,33 @@ struct ConvertFileWritingConsumer: ConvertOutputConsumer, ExternalNodeConsumer {
         }
     }
 
+    private var linkableElementsData = Synchronized(Data())
+    
+    /// Consumes one  linkable element summary produced during a conversion.
+    func consumeIncremental(linkableElementSummary: LinkDestinationSummary) throws {
+        let data = try encode(linkableElementSummary)
+        linkableElementsData.sync {
+            if !$0.isEmpty {
+                $0.append(Data(",".utf8))
+            }
+            $0.append(data)
+        }
+    }
+    
+    /// Finishes consuming the linkable element summaries produced during a conversion.
+    func finishedConsumingLinkElementSummaries() throws {
+        let linkableElementsURL = targetFolder.appendingPathComponent(Self.linkableEntitiesFileName, isDirectory: false)
+        let data = linkableElementsData.sync { accumulatedData in
+            var data = Data()
+            swap(&data, &accumulatedData)
+            data.insert(UTF8.CodeUnit(ascii: "["), at: 0)
+            data.append(UTF8.CodeUnit(ascii: "]"))
+            
+            return data
+        }
+        try fileManager.createFile(at: linkableElementsURL, contents: data)
+    }
+    
     func consume(linkableElementSummaries summaries: [LinkDestinationSummary]) throws {
         let linkableElementsURL = targetFolder.appendingPathComponent(Self.linkableEntitiesFileName, isDirectory: false)
         let data = try encode(summaries)

Sources/SwiftDocCUtilities/ArgumentParsing/Subcommands/Convert.swift

@@ -496,7 +496,7 @@ extension Docc {
 
             // This flag only exist to allow developers to pass the previous '--enable-experimental-...' flag without errors.
             @Flag(name: .customLong("enable-experimental-external-link-support"), help: .hidden)
-            @available(*, deprecated, message: "This deprecated API will be removed after 6.2 is released")
+            @available(*, deprecated, message: "This deprecated API will be removed after 6.3 is released")
             var enableExperimentalLinkHierarchySerialization = false
 
             @Flag(help: .hidden)
@@ -595,7 +595,7 @@ extension Docc {
             set { featureFlags.enableLinkHierarchySerialization = newValue }
         }
 
-        @available(*, deprecated, renamed: "enableLinkHierarchySerialization", message: "Use 'enableLinkHierarchySerialization' instead. This deprecated API will be removed after 6.2 is released")
+        @available(*, deprecated, renamed: "enableLinkHierarchySerialization", message: "Use 'enableLinkHierarchySerialization' instead. This deprecated API will be removed after 6.3 is released")
         public var enableExperimentalLinkHierarchySerialization: Bool {
             get { enableLinkHierarchySerialization }
             set { enableLinkHierarchySerialization = newValue }

Tests/SwiftDocCTests/TestRenderNodeOutputConsumer.swift

@@ -23,6 +23,8 @@ class TestRenderNodeOutputConsumer: ConvertOutputConsumer, ExternalNodeConsumer
 
     func consume(assetsInBundle bundle: DocumentationBundle) throws { }
     func consume(linkableElementSummaries: [LinkDestinationSummary]) throws { }
+    func consumeIncremental(linkableElementSummary: LinkDestinationSummary) throws { }
+    func finishedConsumingLinkElementSummaries() throws { }
     func consume(indexingRecords: [IndexingRecord]) throws { }
     func consume(assets: [RenderReferenceType: [any RenderReference]]) throws { }
     func consume(benchmarks: Benchmark) throws { }