Consistently curate language refined symbols in the language specific places (#854)

* Add source language information to nodes in the path hierarchy

rdar://122300247

* Explicitly track when topics are manually curated.

The previous heuristic of checking if a node has more than 1 parent is incorrect when a symbol with multiple language representations exist in different places in different source languages.

* Fix unrelated bug where topic references didn't encode their navigator title variants

* Fix unrelated bug where `SourceLanguage(id:)` didn't recognize C++ IDs

* Fix unrelated bug where handcrafted test data isn't recognized as default implementation

* Use path hierarchy to source automatic curation for symbols

* Update curation tests that were verifying behavior caused by bugs

* Remove test that verifies an behavior that will never occur

Symbols will never be automatically curated under an article. Manually adjusting topic graph edges makes the test unrealistic.

* Fix hand crafted test data that incorrectly mixed source languages in a single symbol graph

* Fix more hand crafted test data that incorrectly mixed source languages in a single symbol graph

* Change isDisfavoredInCollision into a "special behaviors" option set

* Bring over bug fixes from follow up PR

* Unify check that a node matches a language filter

* Minor grammar fix in comment

Author: d-ronnqvist

Sources/SwiftDocC/Infrastructure/DocumentationContext.swift

@@ -117,6 +117,8 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
     var dataProvider: DocumentationContextDataProvider
 
     /// The graph of all the documentation content and their relationships to each other.
+    ///
+    /// > Important: The topic graph has no awareness of source language specific edges.
     var topicGraph = TopicGraph()
 
     /// User-provided global options for this documentation conversion.
@@ -2320,7 +2322,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
             guard let topicGraphNode = topicGraph.nodeWithReference(reference),
                   let topicGraphParentNode = topicGraph.nodeWithReference(parentReference),
                   // Check that the node hasn't got any parents from manual curation
-                  topicGraph.reverseEdges[reference] == nil
+                  !topicGraphNode.isManuallyCurated
             else { return }
             topicGraph.addEdge(from: topicGraphParentNode, to: topicGraphNode)
             automaticallyCuratedSymbols.append((child: reference, parent: parentReference))
@@ -2386,6 +2388,7 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
                 of: reference,
                 relateNodes: {
                     self.topicGraph.unsafelyAddEdge(source: $0, target: $1)
+                    self.topicGraph.nodes[$1]?.isManuallyCurated = true
                 }
             )
         }
@@ -2573,6 +2576,8 @@ public class DocumentationContext: DocumentationContextDataProviderDelegate {
 
     /// Fetch the child nodes of a documentation node with the given `reference`, optionally filtering to only children of the given `kind`.
     ///
+    /// > Important: The returned list can't be used to determine source language specific children.
+    ///
     /// - Parameters:
     ///   - reference: The reference of the node to fetch children for.
     ///   - kind: An optional documentation node kind to filter the children by.

Sources/SwiftDocC/Infrastructure/Link Resolution/PathHierarchy+Dump.swift

@@ -22,7 +22,7 @@ private extension PathHierarchy.Node {
     func dumpableNode() -> DumpableNode {
         // Each node is printed as 3-layer hierarchy with the child names, their kind disambiguation, and their hash disambiguation.
         return DumpableNode(
-            name: symbol.map { "{ \($0.identifier.precise) : \($0.identifier.interfaceLanguage).\($0.kind.identifier.identifier) }" } ?? "[ \(name) ]",
+            name: symbol.map { "{ \($0.identifier.precise) : \($0.kind.identifier.identifier) [\(languages.map(\.name).joined(separator: ", "))] }" } ?? "[ \(name) ]",
             children: children.sorted(by: \.key).map { (key, disambiguationTree) -> DumpableNode in
                 let grouped = [String: [PathHierarchy.DisambiguationContainer.Element]](grouping: disambiguationTree.storage, by: { $0.kind ?? "_" })
                 return DumpableNode(

Sources/SwiftDocC/Infrastructure/Link Resolution/PathHierarchy+Find.swift

@@ -293,7 +293,7 @@ extension PathHierarchy {
         onlyFindSymbols: Bool,
         rawPathForError: String
     ) throws -> Node {
-        if let favoredMatch = collisions.singleMatch({ $0.node.isDisfavoredInCollision == false }) {
+        if let favoredMatch = collisions.singleMatch({ $0.node.specialBehaviors.contains(.disfavorInLinkCollision) == false }) {
             return favoredMatch.node
         }
         // If a module has the same name as the article root (which is named after the bundle display name) then its possible

Sources/SwiftDocC/Infrastructure/Link Resolution/PathHierarchy+Serialization.swift

@@ -42,7 +42,7 @@ extension PathHierarchy.FileRepresentation {
                     at: identifierMap[node.identifier]!,
                     to: Node(
                         name: node.name,
-                        isDisfavoredInCollision: node.isDisfavoredInCollision,
+                        rawSpecialBehavior: node.specialBehaviors.rawValue,
                         children: node.children.values.flatMap({ tree in
                             var disambiguations = [Node.Disambiguation]()
                             for element in tree.storage where element.node.identifier != nil { // nodes without identifiers can't be found in the tree
@@ -102,7 +102,7 @@ extension PathHierarchy {
         /// A node in the hierarchy.
         struct Node: Codable {
             var name: String
-            var isDisfavoredInCollision: Bool = false
+            var rawSpecialBehavior: Int = 0
             var children: [Disambiguation] = []
             var symbolID: SymbolGraph.Symbol.Identifier?
 
@@ -118,7 +118,7 @@ extension PathHierarchy {
 extension PathHierarchy.FileRepresentation.Node {
     enum CodingKeys: String, CodingKey {
         case name
-        case isDisfavoredInCollision = "disfavored"
+        case rawSpecialBehavior = "disfavored"
         case children
         case symbolID
     }
@@ -127,7 +127,7 @@ extension PathHierarchy.FileRepresentation.Node {
         let container = try decoder.container(keyedBy: CodingKeys.self)
 
         self.name = try container.decode(String.self, forKey: .name)
-        self.isDisfavoredInCollision = try container.decodeIfPresent(Bool.self, forKey: .isDisfavoredInCollision) ?? false
+        self.rawSpecialBehavior = try container.decodeIfPresent(Int.self, forKey: .rawSpecialBehavior) ?? 0
         self.children = try container.decodeIfPresent([Disambiguation].self, forKey: .children) ?? []
         self.symbolID = try container.decodeIfPresent(SymbolGraph.Symbol.Identifier.self, forKey: .symbolID)
     }
@@ -136,7 +136,9 @@ extension PathHierarchy.FileRepresentation.Node {
         var container: KeyedEncodingContainer = encoder.container(keyedBy: CodingKeys.self)
 
         try container.encode(self.name, forKey: .name)
-        try container.encodeIfTrue(self.isDisfavoredInCollision, forKey: .isDisfavoredInCollision)
+        if rawSpecialBehavior > 0 {
+            try container.encode(rawSpecialBehavior, forKey: .rawSpecialBehavior)
+        }
         try container.encodeIfNotEmpty(self.children, forKey: .children)
         try container.encodeIfPresent(symbolID, forKey: .symbolID)
     }

Sources/SwiftDocC/Infrastructure/Link Resolution/PathHierarchy.swift

@@ -68,11 +68,15 @@ struct PathHierarchy {
         var allNodes: [String: [Node]] = [:]
 
         let symbolGraphs = loader.symbolGraphs
-            .sorted(by: { lhs, _ in
-                return !lhs.key.lastPathComponent.contains("@")
+            .map { url, graph in
+                // Only compute the source language for each symbol graph once.
+                (url: url, graph: graph, language: graph.symbols.values.mapFirst(where: { SourceLanguage(id: $0.identifier.interfaceLanguage) }))
+            }
+            .sorted(by: { lhs, rhs in
+                return !lhs.url.lastPathComponent.contains("@")
             })
 
-        for (url, graph) in symbolGraphs {
+        for (url, graph, language) in symbolGraphs {
             let moduleName = graph.module.name
             let moduleNode: Node
 
@@ -84,11 +88,11 @@ struct PathHierarchy {
             } else if let existingModuleNode = roots[moduleName] {
                 moduleNode = existingModuleNode
             } else {
-                let moduleIdentifierLanguage = graph.symbols.values.first?.identifier.interfaceLanguage ?? SourceLanguage.swift.id
+                let moduleIdentifierLanguage = language ?? .swift
                 let moduleSymbol = SymbolGraph.Symbol(
-                    identifier: .init(precise: moduleName, interfaceLanguage: moduleIdentifierLanguage),
+                    identifier: .init(precise: moduleName, interfaceLanguage: moduleIdentifierLanguage.id),
                     names: SymbolGraph.Symbol.Names(title: moduleName, navigator: nil, subHeading: nil, prose: nil),
-                    pathComponents: [moduleName],
+                    pathComponents: [], // Other symbols don't include the module name in their path components.
                     docComment: nil,
                     accessLevel: SymbolGraph.Symbol.AccessControl(rawValue: "public"),
                     kind: SymbolGraph.Symbol.Kind(parsedIdentifier: .module, displayName: moduleKindDisplayName),
@@ -98,24 +102,37 @@ struct PathHierarchy {
                 moduleNode = newModuleNode
                 allNodes[moduleName] = [moduleNode]
             }
+            if let language = language {
+                moduleNode.languages.insert(language)
+            }
 
             var nodes: [String: Node] = [:]
             nodes.reserveCapacity(graph.symbols.count)
             for (id, symbol) in graph.symbols {
                 if let existingNode = allNodes[id]?.first(where: {
-                    // If both identifiers are in the same language, they are the same symbol
+                    // If both identifiers are in the same language, they are the same symbol.
                     $0.symbol!.identifier.interfaceLanguage == symbol.identifier.interfaceLanguage
-                    // Otherwise, if both have the same name and kind their differences doesn't matter for link resolution purposes
-                    || ($0.name == symbol.pathComponents.last && $0.symbol!.kind.identifier == symbol.kind.identifier)
+                    // If both have the same path components and kind, their differences don't matter for link resolution purposes.
+                    || ($0.symbol!.pathComponents == symbol.pathComponents && $0.symbol!.kind.identifier == symbol.kind.identifier)
                 }) {
                     nodes[id] = existingNode
+                    existingNode.languages.insert(language!) // If we have symbols in this graph we have a language as well
                 } else {
                     assert(!symbol.pathComponents.isEmpty, "A symbol should have at least its own name in its path components.")
                     let node = Node(symbol: symbol, name: symbol.pathComponents.last!)
                     // Disfavor synthesized symbols when they collide with other symbol with the same path.
                     // FIXME: Get information about synthesized symbols from SymbolKit https://github.com/apple/swift-docc-symbolkit/issues/58
-                    node.isDisfavoredInCollision = symbol.identifier.precise.contains("::SYNTHESIZED::")
+                    if symbol.identifier.precise.contains("::SYNTHESIZED::") {
+                        node.specialBehaviors = [.disfavorInLinkCollision, .excludeFromAutomaticCuration]
+                    }
                     nodes[id] = node
+                    
+                    if let existing = allNodes[id] {
+                        node.counterpart = existing.first
+                        for other in existing {
+                            other.counterpart = node
+                        }
+                    }
                     allNodes[id, default: []].append(node)
                 }
             }
@@ -154,7 +171,7 @@ struct PathHierarchy {
                 }
                 // Default implementations collide with the protocol requirement that they implement.
                 // Disfavor the default implementation to favor the protocol requirement (or other symbol with the same path).
-                sourceNode.isDisfavoredInCollision = true
+                sourceNode.specialBehaviors = [.disfavorInLinkCollision, .excludeFromAutomaticCuration]
 
                 guard sourceNode.parent == nil else {
                     // This node already has a direct member-of parent. No need to go via the default-implementation-of relationship to find its location in the hierarchy.
@@ -213,15 +230,15 @@ struct PathHierarchy {
                     guard knownDisambiguatedPathComponents != nil else {
                         // If the path hierarchy wasn't passed any "known disambiguated path components" then the sparse/placeholder nodes won't contain any disambiguation.
                         let nodeWithoutSymbol = Node(name: component)
-                        nodeWithoutSymbol.isDisfavoredInCollision = true
+                        nodeWithoutSymbol.specialBehaviors = [.disfavorInLinkCollision, .excludeFromAutomaticCuration]
                         parent.add(child: nodeWithoutSymbol, kind: nil, hash: nil)
                         parent = nodeWithoutSymbol
                         continue
                     }
                     // If the path hierarchy was passed a lookup of "known disambiguation" path components", then it's possible that each path component could contain disambiguation that needs to be parsed.
                     let component = PathParser.parse(pathComponent: component[...])
                     let nodeWithoutSymbol = Node(name: String(component.name))
-                    nodeWithoutSymbol.isDisfavoredInCollision = true
+                    nodeWithoutSymbol.specialBehaviors = [.disfavorInLinkCollision, .excludeFromAutomaticCuration]
                     // Create a spare/placeholder node with the parsed disambiguation for this path component.
                     switch component.disambiguation {
                     case .kindAndHash(kind: let kind, hash: let hash):
@@ -265,8 +282,8 @@ struct PathHierarchy {
                 node.identifier = ResolvedIdentifier()
                 lookup[node.identifier] = node
             }
-            for tree in node.children.values {
-                for element in tree.storage {
+            for container in node.children.values {
+                for element in container.storage {
                     assert(element.node.parent === node, {
                         func describe(_ node: Node?) -> String {
                             guard let node = node else { return "<nil>" }
@@ -402,29 +419,45 @@ extension PathHierarchy {
         fileprivate(set) unowned var parent: Node?
         /// The symbol, if a node has one.
         fileprivate(set) var symbol: SymbolGraph.Symbol?
-        
-        /// If the path hierarchy should disfavor this node in a link collision.
-        ///
-        /// By default, nodes are not disfavored.
+        /// The languages where this node's symbol is represented.
+        fileprivate(set) var languages: Set<SourceLanguage> = []
+        /// The other language representation of this symbol.
         ///
-        /// If a favored node collides with a disfavored node the link will resolve to the favored node without
-        /// requiring any disambiguation. Referencing the disfavored node requires disambiguation.
-        var isDisfavoredInCollision: Bool
+        /// > Note: Swift currently only supports one other language representation (either Objective-C or C++ but not both).
+        fileprivate(set) unowned var counterpart: Node?
+        
+        /// A set of non-standard behaviors that apply to this node.
+        fileprivate(set) var specialBehaviors: SpecialBehaviors
+        
+        /// Options that specify non-standard behaviors of a node.
+        struct SpecialBehaviors: OptionSet {
+            let rawValue: Int
+            
+            /// This node is disfavored in the the case of a link collision.
+            ///
+            /// If a favored node collides with a disfavored node the link will resolve to the favored node without requiring any disambiguation.
+            /// Referencing the disfavored node requires disambiguation unless it's the only match for that link.
+            static let disfavorInLinkCollision = SpecialBehaviors(rawValue: 1 << 0)
+            
+            /// This node is excluded from automatic curation.
+            static let excludeFromAutomaticCuration = SpecialBehaviors(rawValue: 1 << 1)
+        }
 
         /// Initializes a symbol node.
         fileprivate init(symbol: SymbolGraph.Symbol!, name: String) {
             self.symbol = symbol
             self.name = name
             self.children = [:]
-            self.isDisfavoredInCollision = false
+            self.specialBehaviors = []
+            self.languages = [SourceLanguage(id: symbol.identifier.interfaceLanguage)]
         }
 
         /// Initializes a non-symbol node with a given name.
         fileprivate init(name: String) {
             self.symbol = nil
             self.name = name
             self.children = [:]
-            self.isDisfavoredInCollision = false
+            self.specialBehaviors = []
         }
 
         /// Adds a descendant to this node, providing disambiguation information from the node's symbol.
@@ -463,6 +496,10 @@ extension PathHierarchy {
                     element.node.parent = self
                 }
             }
+            
+            if let otherSymbol = other.symbol {
+                languages.insert(SourceLanguage(id: otherSymbol.identifier.interfaceLanguage))
+            }
         }
     }
 }
@@ -640,7 +677,7 @@ extension PathHierarchy {
             } else {
                 node = Node(name: fileNode.name)
             }
-            node.isDisfavoredInCollision = fileNode.isDisfavoredInCollision
+            node.specialBehaviors = .init(rawValue: fileNode.rawSpecialBehavior)
             node.identifier = identifiers[index]
             lookup[node.identifier] = node
         }

Sources/SwiftDocC/Infrastructure/Link Resolution/PathHierarchyBasedLinkResolver.swift

@@ -49,7 +49,6 @@ final class PathHierarchyBasedLinkResolver {
     func traverseSymbolAndParentPairs(_ observe: (_ symbol: ResolvedTopicReference, _ parent: ResolvedTopicReference) -> Void) {
         for (id, node) in pathHierarchy.lookup {
             guard node.symbol != nil else { continue }
-            
             guard let parentID = node.parent?.identifier else { continue }
 
             // Only symbols in the symbol index are added to the reference map.
@@ -58,6 +57,42 @@ final class PathHierarchyBasedLinkResolver {
         }
     }
 
+    /// Returns the direct descendants of the given page that match the given source language filter.
+    ///
+    /// A descendant is included if it has a language representation in at least one of the languages in the given language filter or if the language filter is empty.
+    ///
+    /// - Parameters:
+    ///   - reference: The identifier of the page whose descendants to return.
+    ///   - languagesFilter: A set of source languages to filter descendants against.
+    /// - Returns: The references of each direct descendant that has a language representation in at least one of the given languages.
+    func directDescendants(of reference: ResolvedTopicReference, languagesFilter: Set<SourceLanguage>) -> Set<ResolvedTopicReference> {
+        guard let id = resolvedReferenceMap[reference] else { return [] }
+        let node = pathHierarchy.lookup[id]!
+        
+        func directDescendants(of node: PathHierarchy.Node) -> [ResolvedTopicReference] {
+            return node.children.flatMap { _, container in
+                container.storage.compactMap { element in
+                    guard let childID = element.node.identifier, // Don't include sparse nodes
+                          !element.node.specialBehaviors.contains(.excludeFromAutomaticCuration),
+                          element.node.matches(languagesFilter: languagesFilter)
+                    else {
+                        return nil
+                    }
+                    return resolvedReferenceMap[childID]
+                }
+            }
+        }
+        
+        var results = Set<ResolvedTopicReference>()
+        if node.matches(languagesFilter: languagesFilter) {
+            results.formUnion(directDescendants(of: node))
+        }
+        if let counterpart = node.counterpart, counterpart.matches(languagesFilter: languagesFilter) {
+            results.formUnion(directDescendants(of: counterpart))
+        }
+        return results
+    }
+    
     /// Traverse all symbols of the same kind that have collisions.
     func traverseOverloadedSymbols(_ observe: (_ overloadedSymbols: [ResolvedTopicReference]) throws -> Void) rethrows {
         try pathHierarchy.traverseOverloadedSymbolGroups() { overloadedSymbols in
@@ -315,3 +350,9 @@ private func linkName<S: StringProtocol>(filename: S) -> String {
 
 private let whitespaceAndDashes = CharacterSet.whitespaces
     .union(CharacterSet(charactersIn: "-–—")) // hyphen, en dash, em dash
+
+private extension PathHierarchy.Node {
+    func matches(languagesFilter: Set<SourceLanguage>) -> Bool {
+        languagesFilter.isEmpty || !self.languages.isDisjoint(with: languagesFilter)
+    }
+}