Skip to content

Commit 90a56b8

Browse files
authored
Add experimental command to write generated curation to documentation extension files (#597)
* Add experimental command to write generated curation to doc extensions Also, add experimental flag to in-place add generated curation during a convert action for easier integration with build workflows. rdar://98528457 * Remove unused CLI flag * Fix bug where modified file was output into original location * Update reference to new command * Remove double space in user facing command description * Add flags to limit generated curation to part of a symbol sub hierarchy * Hide experimental convert flag for in-place curation generation * Format top-level docc commands to make it easier to merge new commands * Rephrase "process-*" command abstracts to match "convert" abstract. * Add trailing newline to generated documentation extension content * Make new tests not dependend on the current build directory * Don't write "## Topics" if the file already has topic sections * Add tests that the action writes the expected files * Fail the command if the generated documentation files fail to write * Elaborate on the 'emit-generated-curation' help text * Minor refinements to the 'emit-generated-curation' help text * Add missing line from content of test data * Address code review feedback
1 parent 633dfa2 commit 90a56b8

17 files changed

+1046
-30
lines changed
Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
/*
2+
This source file is part of the Swift.org open source project
3+
4+
Copyright (c) 2024 Apple Inc. and the Swift project authors
5+
Licensed under Apache License v2.0 with Runtime Library Exception
6+
7+
See https://swift.org/LICENSE.txt for license information
8+
See https://swift.org/CONTRIBUTORS.txt for Swift project authors
9+
*/
10+
11+
import Foundation
12+
import SymbolKit
13+
14+
/// A type that writes the auto-generated curation into documentation extension files.
15+
public struct GeneratedCurationWriter {
16+
let context: DocumentationContext
17+
let catalogURL: URL?
18+
let outputURL: URL
19+
let linkResolver: PathHierarchyBasedLinkResolver
20+
21+
public init(
22+
context: DocumentationContext,
23+
catalogURL: URL?,
24+
outputURL: URL
25+
) {
26+
self.context = context
27+
28+
self.catalogURL = catalogURL
29+
self.outputURL = outputURL
30+
31+
self.linkResolver = context.linkResolver.localResolver
32+
}
33+
34+
/// Generates the markdown representation of the auto-generated curation for a given symbol reference.
35+
///
36+
/// - Parameters:
37+
/// - reference: The symbol reference to generate curation text for.
38+
/// - Returns: The auto-generated curation text, or `nil` if this reference has no auto-generated curation.
39+
func defaultCurationText(for reference: ResolvedTopicReference) -> String? {
40+
guard let node = context.documentationCache[reference],
41+
let symbol = node.semantic as? Symbol,
42+
let automaticTopics = try? AutomaticCuration.topics(for: node, withTraits: [], context: context),
43+
!automaticTopics.isEmpty
44+
else {
45+
return nil
46+
}
47+
48+
let relativeLinks = linkResolver.disambiguatedRelativeLinksForDescendants(of: reference)
49+
50+
// Top-level curation has a few special behaviors regarding symbols with different representations in multiple languages.
51+
let isForTopLevelCuration = symbol.kind.identifier == .module
52+
53+
var text = ""
54+
for taskGroup in automaticTopics {
55+
if isForTopLevelCuration, let firstReference = taskGroup.references.first, context.documentationCache[firstReference]?.symbol?.kind.identifier == .typeProperty {
56+
// Skip type properties in top-level curation. It's not clear what's the right place for these symbols are since they exist in
57+
// different places in different source languages (which documentation extensions don't yet have a way of representing).
58+
continue
59+
}
60+
61+
let links: [(link: String, comment: String?)] = taskGroup.references.compactMap { (curatedReference: ResolvedTopicReference) -> (String, String?)? in
62+
guard let linkInfo = relativeLinks[curatedReference] else { return nil }
63+
// If this link contains disambiguation, include a comment with the full symbol declaration to make it easier to know which symbol the link refers to.
64+
var commentText: String?
65+
if linkInfo.hasDisambiguation {
66+
commentText = context.documentationCache[curatedReference]?.symbol?.declarationFragments?.map(\.spelling)
67+
// Replace sequences of whitespace and newlines with a single space
68+
.joined().split(whereSeparator: { $0.isWhitespace || $0.isNewline }).joined(separator: " ")
69+
}
70+
71+
return ("\n- ``\(linkInfo.link)``", commentText.map { " <!-- \($0) -->" })
72+
}
73+
74+
guard !links.isEmpty else { continue }
75+
76+
text.append("\n\n### \(taskGroup.title ?? "<!-- This auto-generated topic has no title -->")\n")
77+
78+
// Calculate the longest link to nicely align all the comments
79+
let longestLink = links.map(\.link.count).max()! // `links` are non-empty so it's safe to force-unwrap `.max()` here
80+
for (link, comment) in links {
81+
if let comment = comment {
82+
text.append(link.padding(toLength: longestLink, withPad: " ", startingAt: 0))
83+
text.append(comment)
84+
} else {
85+
text.append(link)
86+
}
87+
}
88+
}
89+
90+
guard !text.isEmpty else { return nil }
91+
92+
var prefix = "<!-- The content below this line is auto-generated and is redundant. You should either incorporate it into your content above this line or delete it. -->"
93+
94+
// Add "## Topics" to the curation text unless the symbol already had some manual curation.
95+
let hasAnyManualCuration = symbol.topics?.taskGroups.isEmpty == false
96+
if !hasAnyManualCuration {
97+
prefix.append("\n\n## Topics")
98+
}
99+
return "\(prefix)\(text)\n"
100+
}
101+
102+
enum Error: DescribedError {
103+
case symbolLinkNotFound(TopicReferenceResolutionErrorInfo)
104+
105+
var errorDescription: String {
106+
switch self {
107+
case .symbolLinkNotFound(let errorInfo):
108+
var errorMessage = "'--from-symbol <symbol-link>' not found: \(errorInfo.message)"
109+
for solution in errorInfo.solutions {
110+
errorMessage.append("\n\(solution.summary.replacingOccurrences(of: "\n", with: ""))")
111+
}
112+
return errorMessage
113+
}
114+
}
115+
}
116+
117+
/// Generates documentation extension content with a markdown representation of DocC's auto-generated curation.
118+
/// - Parameters:
119+
/// - symbolLink: A link to the symbol whose sub hierarchy the curation writer will descend.
120+
/// - depthLimit: The depth limit of how far the curation writer will descend from its starting point symbol.
121+
/// - Returns: A collection of file URLs and their markdown content.
122+
public func generateDefaultCurationContents(fromSymbol symbolLink: String? = nil, depthLimit: Int? = nil) throws -> [URL: String] {
123+
// Used in documentation extension page titles to reference symbols that don't already have a documentation extension file.
124+
let allAbsoluteLinks = linkResolver.pathHierarchy.disambiguatedAbsoluteLinks()
125+
126+
guard var curationCrawlRoot = linkResolver.modules().first else {
127+
return [:]
128+
}
129+
130+
if let symbolLink = symbolLink {
131+
switch context.linkResolver.resolve(UnresolvedTopicReference(topicURL: .init(symbolPath: symbolLink)), in: curationCrawlRoot, fromSymbolLink: true, context: context) {
132+
case .success(let foundSymbol):
133+
curationCrawlRoot = foundSymbol
134+
case .failure(_, let errorInfo):
135+
throw Error.symbolLinkNotFound(errorInfo)
136+
}
137+
}
138+
139+
var contentsToWrite = [URL: String]()
140+
for (usr, reference) in context.symbolIndex {
141+
// Filter out symbols that aren't in the specified sub hierarchy.
142+
if symbolLink != nil || depthLimit != nil {
143+
guard reference == curationCrawlRoot || context.pathsTo(reference).contains(where: { path in path.suffix(depthLimit ?? .max).contains(curationCrawlRoot)}) else {
144+
continue
145+
}
146+
}
147+
148+
guard let absoluteLink = allAbsoluteLinks[usr], let curationText = defaultCurationText(for: reference) else { continue }
149+
if let catalogURL = catalogURL, let existingURL = context.documentationExtensionURL(for: reference) {
150+
let updatedFileURL: URL
151+
if catalogURL == outputURL {
152+
updatedFileURL = existingURL
153+
} else {
154+
var url = outputURL
155+
let relativeComponents = existingURL.standardizedFileURL.pathComponents.dropFirst(catalogURL.standardizedFileURL.pathComponents.count)
156+
for component in relativeComponents.dropLast() {
157+
url.appendPathComponent(component, isDirectory: true)
158+
}
159+
url.appendPathComponent(relativeComponents.last!, isDirectory: false)
160+
updatedFileURL = url
161+
}
162+
// Append to the end of the file. See if we can avoid reading the existing contents on disk.
163+
var contents = try String(contentsOf: existingURL)
164+
contents.append("\n")
165+
contents.append(curationText)
166+
contentsToWrite[updatedFileURL] = contents
167+
} else {
168+
let relativeReferencePath = reference.url.pathComponents.dropFirst(2).joined(separator: "/")
169+
let fileName = urlReadablePath("/" + relativeReferencePath)
170+
let newFileURL = NodeURLGenerator.fileSafeURL(outputURL.appendingPathComponent("\(fileName).md"))
171+
172+
let contents = """
173+
# ``\(absoluteLink)``
174+
175+
\(curationText)
176+
"""
177+
contentsToWrite[newFileURL] = contents
178+
}
179+
}
180+
181+
return contentsToWrite
182+
}
183+
}

Sources/SwiftDocC/Infrastructure/DocumentationConverter.swift

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
/*
22
This source file is part of the Swift.org open source project
33

4-
Copyright (c) 2021 Apple Inc. and the Swift project authors
4+
Copyright (c) 2021-2023 Apple Inc. and the Swift project authors
55
Licensed under Apache License v2.0 with Runtime Library Exception
66

77
See https://swift.org/LICENSE.txt for license information
@@ -92,6 +92,8 @@ public struct DocumentationConverter: DocumentationConverterProtocol {
9292
/// The source repository where the documentation's sources are hosted.
9393
var sourceRepository: SourceRepository?
9494

95+
var experimentalModifyCatalogWithGeneratedCuration: Bool
96+
9597
/// The identifiers and access level requirements for symbols that have an expanded version of their documentation page if the requirements are met
9698
var symbolIdentifiersWithExpandedDocumentation: [String: ConvertRequest.ExpandedDocumentationRequirements]? = nil
9799

@@ -139,7 +141,8 @@ public struct DocumentationConverter: DocumentationConverterProtocol {
139141
sourceRepository: SourceRepository? = nil,
140142
isCancelled: Synchronized<Bool>? = nil,
141143
diagnosticEngine: DiagnosticEngine = .init(),
142-
symbolIdentifiersWithExpandedDocumentation: [String: ConvertRequest.ExpandedDocumentationRequirements]? = nil
144+
symbolIdentifiersWithExpandedDocumentation: [String: ConvertRequest.ExpandedDocumentationRequirements]? = nil,
145+
experimentalModifyCatalogWithGeneratedCuration: Bool = false
143146
) {
144147
self.rootURL = documentationBundleURL
145148
self.emitDigest = emitDigest
@@ -156,6 +159,7 @@ public struct DocumentationConverter: DocumentationConverterProtocol {
156159
self.isCancelled = isCancelled
157160
self.diagnosticEngine = diagnosticEngine
158161
self.symbolIdentifiersWithExpandedDocumentation = symbolIdentifiersWithExpandedDocumentation
162+
self.experimentalModifyCatalogWithGeneratedCuration = experimentalModifyCatalogWithGeneratedCuration
159163

160164
// Inject current platform versions if provided
161165
if let currentPlatforms = currentPlatforms {
@@ -246,6 +250,16 @@ public struct DocumentationConverter: DocumentationConverterProtocol {
246250
// For now, we only support one bundle.
247251
let bundle = bundles.first!
248252

253+
if experimentalModifyCatalogWithGeneratedCuration, let catalogURL = rootURL {
254+
let writer = GeneratedCurationWriter(context: context, catalogURL: catalogURL, outputURL: catalogURL)
255+
let curation = try writer.generateDefaultCurationContents()
256+
for (url, updatedContent) in curation {
257+
guard let data = updatedContent.data(using: .utf8) else { continue }
258+
try? FileManager.default.createDirectory(at: url.deletingLastPathComponent(), withIntermediateDirectories: true, attributes: nil)
259+
try? data.write(to: url, options: .atomic)
260+
}
261+
}
262+
249263
guard !context.problems.containsErrors else {
250264
if emitDigest {
251265
try outputConsumer.consume(problems: context.problems)

Sources/SwiftDocC/Infrastructure/Link Resolution/PathHierarchy+DisambiguatedPaths.swift

Lines changed: 77 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,18 +22,90 @@ extension PathHierarchy {
2222
/// - Parameters:
2323
/// - includeDisambiguationForUnambiguousChildren: Whether or not descendants unique to a single collision should maintain the containers disambiguation.
2424
/// - includeLanguage: Whether or not kind disambiguation information should include the source language.
25-
/// - Returns: A map of unique identifier strings to disambiguated file paths
25+
/// - Returns: A map of unique identifier strings to disambiguated file paths.
2626
func caseInsensitiveDisambiguatedPaths(
2727
includeDisambiguationForUnambiguousChildren: Bool = false,
2828
includeLanguage: Bool = false
2929
) -> [String: String] {
30+
return disambiguatedPaths(
31+
caseSensitive: false,
32+
transformToFileNames: true,
33+
includeDisambiguationForUnambiguousChildren: includeDisambiguationForUnambiguousChildren,
34+
includeLanguage: includeLanguage
35+
)
36+
}
37+
38+
/// Determines the disambiguated relative links of all the direct descendants of the given node.
39+
///
40+
/// - Parameters:
41+
/// - nodeID: The identifier of the node to determine direct descendant links for.
42+
/// - Returns: A map if node identifiers to pairs of links and flags indicating if the link is disambiguated or not.
43+
func disambiguatedChildLinks(of nodeID: ResolvedIdentifier) -> [ResolvedIdentifier: (link: String, hasDisambiguation: Bool)] {
44+
let node = lookup[nodeID]!
45+
46+
var gathered = [(symbolID: String, (link: String, hasDisambiguation: Bool, id: ResolvedIdentifier, isSwift: Bool))]()
47+
for (_, tree) in node.children {
48+
let disambiguatedChildren = tree.disambiguatedValuesWithCollapsedUniqueSymbols(includeLanguage: false)
49+
50+
for (node, disambiguation) in disambiguatedChildren {
51+
guard let id = node.identifier, let symbolID = node.symbol?.identifier.precise else { continue }
52+
let suffix = disambiguation.makeSuffix()
53+
gathered.append((
54+
symbolID: symbolID, (
55+
link: node.name + suffix,
56+
hasDisambiguation: !suffix.isEmpty,
57+
id: id,
58+
isSwift: node.symbol?.identifier.interfaceLanguage == "swift"
59+
)
60+
))
61+
}
62+
}
63+
64+
// If a symbol node exist in multiple languages, prioritize the Swift variant.
65+
let uniqueSymbolValues = Dictionary(gathered, uniquingKeysWith: { lhs, rhs in lhs.isSwift ? lhs : rhs })
66+
.values.map({ ($0.id, ($0.link, $0.hasDisambiguation)) })
67+
return .init(uniqueKeysWithValues: uniqueSymbolValues)
68+
}
69+
70+
/// Determines the least disambiguated links for all symbols in the path hierarchy.
71+
///
72+
/// - Returns: A map of unique identifier strings to disambiguated links.
73+
func disambiguatedAbsoluteLinks() -> [String: String] {
74+
return disambiguatedPaths(
75+
caseSensitive: true,
76+
transformToFileNames: false,
77+
includeDisambiguationForUnambiguousChildren: false,
78+
includeLanguage: false
79+
)
80+
}
81+
82+
private func disambiguatedPaths(
83+
caseSensitive: Bool,
84+
transformToFileNames: Bool,
85+
includeDisambiguationForUnambiguousChildren: Bool,
86+
includeLanguage: Bool
87+
) -> [String: String] {
88+
let nameTransform: (String) -> String
89+
if transformToFileNames {
90+
nameTransform = symbolFileName(_:)
91+
} else {
92+
nameTransform = { $0 }
93+
}
94+
3095
func descend(_ node: Node, accumulatedPath: String) -> [(String, (String, Bool))] {
3196
var results: [(String, (String, Bool))] = []
32-
let caseInsensitiveChildren = [String: DisambiguationContainer](node.children.map { (symbolFileName($0.key.lowercased()), $0.value) }, uniquingKeysWith: { $0.merge(with: $1) })
97+
let children = [String: DisambiguationContainer](node.children.map {
98+
var name = $0.key
99+
if !caseSensitive {
100+
name = name.lowercased()
101+
}
102+
return (nameTransform(name), $0.value)
103+
}, uniquingKeysWith: { $0.merge(with: $1) })
33104

34-
for (_, tree) in caseInsensitiveChildren {
105+
for (_, tree) in children {
35106
let disambiguatedChildren = tree.disambiguatedValuesWithCollapsedUniqueSymbols(includeLanguage: includeLanguage)
36107
let uniqueNodesWithChildren = Set(disambiguatedChildren.filter { $0.disambiguation.value() != nil && !$0.value.children.isEmpty }.map { $0.value.symbol?.identifier.precise })
108+
37109
for (node, disambiguation) in disambiguatedChildren {
38110
var path: String
39111
if node.identifier == nil && disambiguatedChildren.count == 1 {
@@ -48,9 +120,9 @@ extension PathHierarchy {
48120
if hash != "_" {
49121
knownDisambiguation += "-\(hash)"
50122
}
51-
path = accumulatedPath + "/" + symbolFileName(node.name) + knownDisambiguation
123+
path = accumulatedPath + "/" + nameTransform(node.name) + knownDisambiguation
52124
} else {
53-
path = accumulatedPath + "/" + symbolFileName(node.name)
125+
path = accumulatedPath + "/" + nameTransform(node.name)
54126
}
55127
if let symbol = node.symbol {
56128
results.append(

Sources/SwiftDocC/Infrastructure/Link Resolution/PathHierarchyBasedLinkResolver.swift

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,26 @@ final class PathHierarchyBasedLinkResolver {
270270
}
271271
return result
272272
}
273+
274+
// MARK: Links
275+
276+
/// Determines the disambiguated relative links of all the direct descendants of the given page.
277+
///
278+
/// - Parameters:
279+
/// - reference: The identifier of the page whose descendants to generate relative links for.
280+
/// - Returns: A map topic references to pairs of links and flags indicating if the link is disambiguated or not.
281+
func disambiguatedRelativeLinksForDescendants(of reference: ResolvedTopicReference) -> [ResolvedTopicReference: (link: String, hasDisambiguation: Bool)] {
282+
guard let nodeID = resolvedReferenceMap[reference] else { return [:] }
283+
284+
let links = pathHierarchy.disambiguatedChildLinks(of: nodeID)
285+
var result = [ResolvedTopicReference: (link: String, hasDisambiguation: Bool)]()
286+
result.reserveCapacity(links.count)
287+
for (id, link) in links {
288+
guard let reference = resolvedReferenceMap[id] else { continue }
289+
result[reference] = link
290+
}
291+
return result
292+
}
273293
}
274294

275295
/// Creates a more writable version of an articles file name for use in documentation links.

0 commit comments

Comments
 (0)