Emit language specific topic sections in the markdown representation of automatic curation (#856)

* Simplify creation of disambiguated paths

* Generate language specific curation markdown

rdar://124527905

* Stop using the topic graph to determine automatic curation

* Conform SourceLanguage to Comparable

Author: d-ronnqvist

Sources/SwiftDocC/Catalog Processing/GeneratedCurationWriter.swift

@@ -39,25 +39,15 @@ public struct GeneratedCurationWriter {
     func defaultCurationText(for reference: ResolvedTopicReference) -> String? {
         guard let node = context.documentationCache[reference],
               let symbol = node.semantic as? Symbol,
-              let automaticTopics = try? AutomaticCuration.topics(for: node, withTraits: [], context: context),
-              !automaticTopics.isEmpty
+              let taskGroups = taskGroupsToWrite(for: node)
         else {
             return nil
         }
 
         let relativeLinks = linkResolver.disambiguatedRelativeLinksForDescendants(of: reference)
 
-        // Top-level curation has a few special behaviors regarding symbols with different representations in multiple languages.
-        let isForTopLevelCuration = symbol.kind.identifier == .module
-        
         var text = ""
-        for taskGroup in automaticTopics {
-            if isForTopLevelCuration, let firstReference = taskGroup.references.first, context.documentationCache[firstReference]?.symbol?.kind.identifier == .typeProperty {
-                // Skip type properties in top-level curation. It's not clear what's the right place for these symbols are since they exist in
-                // different places in different source languages (which documentation extensions don't yet have a way of representing).
-                continue
-            }
-            
+        for taskGroup in taskGroups {
             let links: [(link: String, comment: String?)] = taskGroup.references.compactMap { (curatedReference: ResolvedTopicReference) -> (String, String?)? in
                 guard let linkInfo = relativeLinks[curatedReference] else { return nil }
                 // If this link contains disambiguation, include a comment with the full symbol declaration to make it easier to know which symbol the link refers to.
@@ -73,7 +63,10 @@ public struct GeneratedCurationWriter {
 
             guard !links.isEmpty else { continue }
 
-            text.append("\n\n### \(taskGroup.title ?? "<!-- This auto-generated topic has no title -->")\n")
+            text.append("\n\n### \(taskGroup.title)\n")
+            if let language = taskGroup.languageFilter {
+                text.append("\n@SupportedLanguage(\(language.taskGroupID))\n")
+            }
 
             // Calculate the longest link to nicely align all the comments
             let longestLink = links.map(\.link.count).max()! // `links` are non-empty so it's safe to force-unwrap `.max()` here
@@ -180,4 +173,95 @@ public struct GeneratedCurationWriter {
 
         return contentsToWrite
     }
+    
+    private struct GeneratedTaskGroup: Equatable {
+        var title: String
+        var references: [ResolvedTopicReference]
+        var languageFilter: SourceLanguage?
+    }
+    
+    private func taskGroupsToWrite(for node: DocumentationNode) -> [GeneratedTaskGroup]? {
+        // Perform source-language specific curation in a stable order
+        let languagesToCurate = node.availableSourceLanguages.sorted()
+        var topicsByLanguage = [SourceLanguage: [AutomaticCuration.TaskGroup]]()
+        for language in languagesToCurate {
+            topicsByLanguage[language] = try? AutomaticCuration.topics(for: node, withTraits: [.init(interfaceLanguage: language.id)], context: context)
+        }
+        
+        guard topicsByLanguage.count > 1 else {
+            // If this node doesn't have curation in more than one language, return that curation _without_ a language filter.
+            return topicsByLanguage.first?.value.map { .init(title: $0.title! /* Automatically-generated task groups always have a title. */, references: $0.references) }
+        }
+        
+        // Checks if a node for the given reference is only available in the given source language.
+        func isOnlyAvailableIn(_ reference: ResolvedTopicReference, _ language: SourceLanguage) -> Bool {
+            context.documentationCache[reference]?.availableSourceLanguages == [language]
+        }
+        
+        // This is defined as an inner function so that the taskGroups-loop can return to finish processing task groups for that symbol kind.
+        func addProcessedTaskGroups(_ symbolKind: SymbolGraph.Symbol.KindIdentifier, into result: inout [GeneratedTaskGroup]) {
+            let title = AutomaticCuration.groupTitle(for: symbolKind)
+            let taskGroups: [GeneratedTaskGroup] = languagesToCurate.compactMap { language in
+                topicsByLanguage[language]?
+                    .first(where: { $0.title == title })
+                    .map { (title, references) in
+                        // Automatically-generated task groups always have a title.
+                        GeneratedTaskGroup(title: title!, references: references, languageFilter: language)
+                    }
+            }
+            guard taskGroups.count > 1 else {
+                if let taskGroup = taskGroups.first {
+                    if taskGroup.references.allSatisfy({ isOnlyAvailableIn($0, taskGroup.languageFilter!) }) {
+                        // These symbols are all only available in one source language. We can omit the language filter.
+                        result.append(.init(title: title, references: taskGroup.references))
+                    } else {
+                        // Add the task group as-is, with its language filter.
+                        result.append(taskGroup)
+                    }
+                }
+                return
+            }
+            
+            // Check if the source-language specific task groups need to be emitted individually
+            for taskGroup in taskGroups {
+                // Gather all the references for the other task groups
+                let otherReferences = taskGroups.reduce(into: Set(), { accumulator, group in
+                    guard group != taskGroup else { return }
+                    accumulator.formUnion(group.references)
+                })
+                let uniqueReferences = Set(taskGroup.references).subtracting(otherReferences)
+                
+                guard uniqueReferences.isEmpty || uniqueReferences.allSatisfy({ isOnlyAvailableIn($0, taskGroup.languageFilter!) }) else {
+                    // If at least one of the task groups contains a a language-refined symbol that's not in the other task groups, then emit all task groups individually
+                    result.append(contentsOf: taskGroups)
+                    return
+                }
+            }
+            
+            // Otherwise, if the task groups all contain the same symbols or only contain single-language symbols, emit a combined task group without a language filter.
+            // When DocC renders this task group it will filter out the single-language symbols, producing correct and consistent results in all language variants without
+            // needing to repeat the task groups with individual language filters.
+            result.append(.init(
+                title: title,
+                references: taskGroups.reduce(into: Set(), { $0.formUnion($1.references) }).sorted(by: \.path)
+            ))
+        }
+        
+        var result = [GeneratedTaskGroup]()
+        for symbolKind in AutomaticCuration.groupKindOrder {
+            addProcessedTaskGroups(symbolKind, into: &result)
+        }
+        return result
+    }
+}
+
+private extension SourceLanguage {
+    var taskGroupID: String {
+        switch self {
+        case .objectiveC:
+            return "objc"
+        default:
+            return self.linkDisambiguationID
+        }
+    }
 }

Sources/SwiftDocC/Infrastructure/Link Resolution/PathHierarchy+DisambiguatedPaths.swift

@@ -44,23 +44,31 @@ extension PathHierarchy {
         let node = lookup[nodeID]!
 
         var gathered = [(symbolID: String, (link: String, hasDisambiguation: Bool, id: ResolvedIdentifier, isSwift: Bool))]()
-        for (_, tree) in node.children {
-            let disambiguatedChildren = tree.disambiguatedValuesWithCollapsedUniqueSymbols(includeLanguage: false)
-            
-            for (node, disambiguation) in disambiguatedChildren {
-                guard let id = node.identifier, let symbolID = node.symbol?.identifier.precise else { continue }
-                let suffix = disambiguation.makeSuffix()
-                gathered.append((
-                    symbolID: symbolID, (
-                        link: node.name + suffix,
-                        hasDisambiguation: !suffix.isEmpty,
-                        id: id,
-                        isSwift: node.symbol?.identifier.interfaceLanguage == "swift"
-                    )
-                ))
+        
+        func gatherLinksFrom(_ containers: some Sequence<DisambiguationContainer>) {
+            for container in containers {
+                let disambiguatedChildren = container.disambiguatedValuesWithCollapsedUniqueSymbols(includeLanguage: false)
+                
+                for (node, disambiguation) in disambiguatedChildren {
+                    guard let id = node.identifier, let symbolID = node.symbol?.identifier.precise else { continue }
+                    let suffix = disambiguation.makeSuffix()
+                    gathered.append((
+                        symbolID: symbolID, (
+                            link: node.name + suffix,
+                            hasDisambiguation: !suffix.isEmpty,
+                            id: id,
+                            isSwift: node.symbol?.identifier.interfaceLanguage == "swift"
+                        )
+                    ))
+                }
             }
         }
 
+        gatherLinksFrom(node.children.values)
+        if let counterpart = node.counterpart {
+            gatherLinksFrom(counterpart.children.values)
+        }
+        
         // If a symbol node exist in multiple languages, prioritize the Swift variant.
         let uniqueSymbolValues = Dictionary(gathered, uniquingKeysWith: { lhs, rhs in lhs.isSwift ? lhs : rhs })
             .values.map({ ($0.id, ($0.link, $0.hasDisambiguation)) })
@@ -92,8 +100,8 @@ extension PathHierarchy {
             nameTransform = { $0 }
         }
 
-        func descend(_ node: Node, accumulatedPath: String) -> [(String, (String, Bool))] {
-            var results: [(String, (String, Bool))] = []
+        func descend(_ node: Node, accumulatedPath: String) -> [String: String] {
+            var innerPathsByUSR: [String: String] = [:]
             let children = [String: DisambiguationContainer](node.children.map {
                 var name = $0.key
                 if !caseSensitive {
@@ -102,8 +110,8 @@ extension PathHierarchy {
                 return (nameTransform(name), $0.value)
             }, uniquingKeysWith: { $0.merge(with: $1) })
 
-            for (_, tree) in children {
-                let disambiguatedChildren = tree.disambiguatedValuesWithCollapsedUniqueSymbols(includeLanguage: includeLanguage)
+            for (_, container) in children {
+                let disambiguatedChildren = container.disambiguatedValuesWithCollapsedUniqueSymbols(includeLanguage: includeLanguage)
                 let uniqueNodesWithChildren = Set(disambiguatedChildren.filter { $0.disambiguation.value() != nil && !$0.value.children.isEmpty }.map { $0.value.symbol?.identifier.precise })
 
                 for (node, disambiguation) in disambiguatedChildren {
@@ -112,7 +120,7 @@ extension PathHierarchy {
                         // When descending through placeholder nodes, we trust that the known disambiguation
                         // that they were created with is necessary.
                         var knownDisambiguation = ""
-                        let element = tree.storage.first!
+                        let element = container.storage.first!
                         if let kind = element.kind {
                             knownDisambiguation += "-\(kind)"
                         }
@@ -123,37 +131,39 @@ extension PathHierarchy {
                     } else {
                         path = accumulatedPath + "/" + nameTransform(node.name)
                     }
-                    if let symbol = node.symbol {
-                        results.append(
-                            (symbol.identifier.precise, (path + disambiguation.makeSuffix(), symbol.identifier.interfaceLanguage == "swift"))
-                        )
+                    if let symbol = node.symbol,
+                       // If a symbol node exist in multiple languages, prioritize the Swift variant.
+                       node.counterpart == nil || symbol.identifier.interfaceLanguage == "swift"
+                    {
+                        innerPathsByUSR[symbol.identifier.precise] = path + disambiguation.makeSuffix()
                     }
                     if includeDisambiguationForUnambiguousChildren || uniqueNodesWithChildren.count > 1 {
                         path += disambiguation.makeSuffix()
                     }
-                    results += descend(node, accumulatedPath: path)
+                    innerPathsByUSR.merge(descend(node, accumulatedPath: path), uniquingKeysWith: { currentPath, newPath in
+                        assertionFailure("Should only have gathered one path per symbol ID. Found \(currentPath) and \(newPath) for the same USR.")
+                        return currentPath
+                    })
                 }
             }
-            return results
+            return innerPathsByUSR
         }
 
-        var gathered: [(String, (String, Bool))] = []
+        var pathsByUSR: [String: String] = [:]
 
         for node in modules {
-            let path = "/" + node.name
-            gathered.append(
-                (node.name, (path, node.symbol == nil || node.symbol!.identifier.interfaceLanguage == "swift"))
-            )
-            gathered += descend(node, accumulatedPath: path)
+            let modulePath = "/" + node.name
+            pathsByUSR[node.name] = modulePath
+            pathsByUSR.merge(descend(node, accumulatedPath: modulePath), uniquingKeysWith: { currentPath, newPath in
+                assertionFailure("Should only have gathered one path per symbol ID. Found \(currentPath) and \(newPath) for the same USR.")
+                return currentPath
+            })
         }
 
-        // If a symbol node exist in multiple languages, prioritize the Swift variant.
-        let result = [String: (String, Bool)](gathered, uniquingKeysWith: { lhs, rhs in lhs.1 ? lhs : rhs }).mapValues({ $0.0 })
-        
         assert(
-            Set(result.values).count == result.keys.count,
+            Set(pathsByUSR.values).count == pathsByUSR.keys.count,
             {
-                let collisionDescriptions = result
+                let collisionDescriptions = pathsByUSR
                     .reduce(into: [String: [String]](), { $0[$1.value, default: []].append($1.key) })
                     .filter({ $0.value.count > 1 })
                     .map { "\($0.key)\n\($0.value.map({ "  " + $0 }).joined(separator: "\n"))" }
@@ -164,7 +174,7 @@ extension PathHierarchy {
             }()
         )
 
-        return result
+        return pathsByUSR
     }
 }
 

Sources/SwiftDocC/Infrastructure/Symbol Graph/GeneratedDocumentationTopics.swift

@@ -175,18 +175,7 @@ enum GeneratedDocumentationTopics {
             )
         }
 
-        // Create a temp node in order to generate the automatic curation
-        let temporaryCollectionNode = DocumentationNode(
-            reference: collectionReference,
-            kind: .collectionGroup,
-            sourceLanguage: automaticCurationSourceLanguage,
-            availableSourceLanguages: automaticCurationSourceLanguages,
-            name: DocumentationNode.Name.conceptual(title: title),
-            markup: Document(parsing: ""),
-            semantic: Article(markup: nil, metadata: nil, redirects: nil, options: [:])
-        )
-        
-        let collectionTaskGroups = try AutomaticCuration.topics(for: temporaryCollectionNode, withTraits: [], context: context)
+        let collectionTaskGroups = try AutomaticCuration.topics(for: identifiers, inInheritedSymbolsAPICollection: true, withTraits: [], context: context)
             .map { taskGroup in
                 AutomaticTaskGroupSection(
                     // Force-unwrapping the title since automatically-generated task groups always have a title.