Skip to content

Commit 9e7b896

Browse files
MpdreamzMpdreamz
authored
Add anchors to link json artifact (#353)
* Add anchors to link json artifact Refactored `LinkReference` to use a dictionary for better anchor tracking and adjusted anchor validations. Added extensive test coverage, revamped test setup, and improved generator logic for markdown files.``` * rename union creation methods * add license headers * make links on object holding anchors * dotnet format
1 parent 20a79c9 commit 9e7b896

File tree

14 files changed

+341
-88
lines changed

14 files changed

+341
-88
lines changed

‎src/Elastic.Markdown/DocumentationGenerator.cs‎

Lines changed: 29 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,12 @@ ILoggerFactory logger
5050
}
5151

5252

53-
public async Task ResolveDirectoryTree(Cancel ctx) =>
53+
public async Task ResolveDirectoryTree(Cancel ctx)
54+
{
55+
_logger.LogInformation("Resolving tree");
5456
await DocumentationSet.Tree.Resolve(ctx);
57+
_logger.LogInformation("Resolved tree");
58+
}
5559

5660
public async Task GenerateAll(Cancel ctx)
5761
{
@@ -62,11 +66,30 @@ public async Task GenerateAll(Cancel ctx)
6266
if (CompilationNotNeeded(generationState, out var offendingFiles, out var outputSeenChanges))
6367
return;
6468

65-
_logger.LogInformation("Resolving tree");
6669
await ResolveDirectoryTree(ctx);
67-
_logger.LogInformation("Resolved tree");
6870

71+
await ProcessDocumentationFiles(offendingFiles, outputSeenChanges, ctx);
72+
73+
await ExtractEmbeddedStaticResources(ctx);
74+
75+
76+
_logger.LogInformation($"Completing diagnostics channel");
77+
Context.Collector.Channel.TryComplete();
6978

79+
_logger.LogInformation($"Generating documentation compilation state");
80+
await GenerateDocumentationState(ctx);
81+
_logger.LogInformation($"Generating links.json");
82+
await GenerateLinkReference(ctx);
83+
84+
_logger.LogInformation($"Completing diagnostics channel");
85+
86+
await Context.Collector.StopAsync(ctx);
87+
88+
_logger.LogInformation($"Completed diagnostics channel");
89+
}
90+
91+
private async Task ProcessDocumentationFiles(HashSet<string> offendingFiles, DateTimeOffset outputSeenChanges, Cancel ctx)
92+
{
7093
var processedFileCount = 0;
7194
var exceptionCount = 0;
7295
_ = Context.Collector.StartAsync(ctx);
@@ -91,7 +114,10 @@ await Parallel.ForEachAsync(DocumentationSet.Files, ctx, async (file, token) =>
91114
if (processedFiles % 100 == 0)
92115
_logger.LogInformation($"-> Handled {processedFiles} files");
93116
});
117+
}
94118

119+
private async Task ExtractEmbeddedStaticResources(Cancel ctx)
120+
{
95121
_logger.LogInformation($"Copying static files to output directory");
96122
var embeddedStaticFiles = Assembly.GetExecutingAssembly()
97123
.GetManifestResourceNames()
@@ -111,21 +137,6 @@ await Parallel.ForEachAsync(DocumentationSet.Files, ctx, async (file, token) =>
111137
await resourceStream.CopyToAsync(stream, ctx);
112138
_logger.LogInformation($"Copied static embedded resource {path}");
113139
}
114-
115-
116-
_logger.LogInformation($"Completing diagnostics channel");
117-
Context.Collector.Channel.TryComplete();
118-
119-
_logger.LogInformation($"Generating documentation compilation state");
120-
await GenerateDocumentationState(ctx);
121-
_logger.LogInformation($"Generating links.json");
122-
await GenerateLinkReference(ctx);
123-
124-
_logger.LogInformation($"Completing diagnostics channel");
125-
126-
await Context.Collector.StopAsync(ctx);
127-
128-
_logger.LogInformation($"Completed diagnostics channel");
129140
}
130141

131142
private async Task ProcessFile(HashSet<string> offendingFiles, DocumentationFile file, DateTimeOffset outputSeenChanges, CancellationToken token)

‎src/Elastic.Markdown/IO/Discovery/GitCheckoutInformation.cs‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,19 +39,18 @@ public string? RepositoryName
3939
// manual read because libgit2sharp is not yet AOT ready
4040
public static GitCheckoutInformation Create(IFileSystem fileSystem)
4141
{
42-
// filesystem is not real so return a dummy
43-
var fakeRef = Guid.NewGuid().ToString().Substring(0, 16);
4442
if (fileSystem is not FileSystem)
4543
{
4644
return new GitCheckoutInformation
4745
{
48-
Branch = $"test-{fakeRef}",
46+
Branch = $"test-e35fcb27-5f60-4e",
4947
Remote = "elastic/docs-builder",
50-
Ref = fakeRef,
48+
Ref = "e35fcb27-5f60-4e",
5149
RepositoryName = "docs-builder"
5250
};
5351
}
5452

53+
var fakeRef = Guid.NewGuid().ToString()[..16];
5554
var gitConfig = Git(".git/config");
5655
if (!gitConfig.Exists)
5756
return Unavailable;

‎src/Elastic.Markdown/IO/MarkdownFile.cs‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,8 @@ public string? NavigationTitle
6363
private readonly Dictionary<string, PageTocItem> _tableOfContent = new(StringComparer.OrdinalIgnoreCase);
6464
public IReadOnlyDictionary<string, PageTocItem> TableOfContents => _tableOfContent;
6565

66-
private readonly HashSet<string> _additionalLabels = new(StringComparer.OrdinalIgnoreCase);
67-
public IReadOnlySet<string> AdditionalLabels => _additionalLabels;
66+
private readonly HashSet<string> _anchors = new(StringComparer.OrdinalIgnoreCase);
67+
public IReadOnlySet<string> Anchors => _anchors;
6868

6969
public string FilePath { get; }
7070
public string FileName { get; }
@@ -171,22 +171,22 @@ private void ReadDocumentInstructions(MarkdownDocument document)
171171
Slug = (h.Item2 ?? h.Item1).Slugify()
172172
})
173173
.ToList();
174+
174175
_tableOfContent.Clear();
175176
foreach (var t in contents)
176177
_tableOfContent[t.Slug] = t;
177178

178-
var labels = document.Descendants<DirectiveBlock>()
179+
var anchors = document.Descendants<DirectiveBlock>()
179180
.Select(b => b.CrossReferenceName)
180181
.Where(l => !string.IsNullOrWhiteSpace(l))
181182
.Select(s => s.Slugify())
182183
.Concat(document.Descendants<InlineAnchor>().Select(a => a.Anchor))
184+
.Concat(_tableOfContent.Values.Select(t => t.Slug))
185+
.Where(anchor => !string.IsNullOrEmpty(anchor))
183186
.ToArray();
184187

185-
foreach (var label in labels)
186-
{
187-
if (!string.IsNullOrEmpty(label))
188-
_additionalLabels.Add(label);
189-
}
188+
foreach (var label in anchors)
189+
_anchors.Add(label);
190190

191191
_instructionsParsed = true;
192192
}

‎src/Elastic.Markdown/IO/State/LinkReference.cs‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,13 @@
77

88
namespace Elastic.Markdown.IO.State;
99

10+
public record LinkMetadata
11+
{
12+
[JsonPropertyName("anchors")]
13+
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
14+
public required string[]? Anchors { get; init; } = [];
15+
}
16+
1017
public record LinkReference
1118
{
1219
[JsonPropertyName("origin")]
@@ -15,8 +22,9 @@ public record LinkReference
1522
[JsonPropertyName("url_path_prefix")]
1623
public required string? UrlPathPrefix { get; init; }
1724

25+
/// Mapping of relative filepath and all the page's anchors for deeplinks
1826
[JsonPropertyName("links")]
19-
public required string[] Links { get; init; } = [];
27+
public required Dictionary<string, LinkMetadata> Links { get; init; } = [];
2028

2129
[JsonPropertyName("cross_links")]
2230
public required string[] CrossLinks { get; init; } = [];
@@ -25,7 +33,12 @@ public static LinkReference Create(DocumentationSet set)
2533
{
2634
var crossLinks = set.Context.Collector.CrossLinks.ToHashSet().ToArray();
2735
var links = set.MarkdownFiles.Values
28-
.Select(m => m.RelativePath).ToArray();
36+
.Select(m => (m.RelativePath, m.Anchors))
37+
.ToDictionary(k => k.RelativePath, v =>
38+
{
39+
var anchors = v.Anchors.Count == 0 ? null : v.Anchors.ToArray();
40+
return new LinkMetadata { Anchors = anchors };
41+
});
2942
return new LinkReference
3043
{
3144
UrlPathPrefix = set.Context.UrlPathPrefix,

‎src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -135,11 +135,9 @@ public override bool Match(InlineProcessor processor, ref StringSlice slice)
135135

136136
if (!string.IsNullOrEmpty(anchor))
137137
{
138-
if (markdown == null || (!markdown.TableOfContents.TryGetValue(anchor, out var heading)
139-
&& !markdown.AdditionalLabels.Contains(anchor)))
138+
if (markdown == null || !markdown.Anchors.Contains(anchor))
140139
processor.EmitError(line, column, length, $"`{anchor}` does not exist in {markdown?.FileName}.");
141-
142-
else if (link.FirstChild == null && heading != null)
140+
else if (link.FirstChild == null && markdown.TableOfContents.TryGetValue(anchor, out var heading))
143141
title += " > " + heading.Heading;
144142

145143
}

‎tests/Elastic.Markdown.Tests/DocSet/LinkReferenceTests.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ public void EmitsLinks() =>
2626

2727
[Fact]
2828
public void ShouldNotIncludeSnippets() =>
29-
Reference.Links.Should().NotContain(l => l.Contains("_snippets/"));
29+
Reference.Links.Should().NotContain(l => l.Key.Contains("_snippets/"));
3030
}
3131

3232
public class GitCheckoutInformationTests(ITestOutputHelper output) : NavigationTestsBase(output)

‎tests/authoring/Container/DefinitionLists.fs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ This is my `definition`
4141
"""
4242

4343
[<Fact>]
44-
let ``validate HTML`` () =
44+
let ``validate HTML 2`` () =
4545
markdown |> convertsToHtml """
4646
<dl>
4747
<dt>This is my <code>definition</code> </dt>
@@ -56,4 +56,4 @@ This is my `definition`
5656
</dl>
5757
"""
5858
[<Fact>]
59-
let ``has no errors`` () = markdown |> hasNoErrors
59+
let ``has no errors 2`` () = markdown |> hasNoErrors

‎tests/authoring/Framework/ErrorCollectorAssertions.fs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,13 @@ open Swensen.Unquote
1414
module DiagnosticsCollectorAssertions =
1515

1616
[<DebuggerStepThrough>]
17-
let hasNoErrors (actual: GenerateResult) =
17+
let hasNoErrors (actual: Lazy<GeneratorResults>) =
18+
let actual = actual.Value
1819
test <@ actual.Context.Collector.Errors = 0 @>
1920

2021
[<DebuggerStepThrough>]
21-
let hasError (expected: string) (actual: GenerateResult) =
22+
let hasError (expected: string) (actual: Lazy<GeneratorResults>) =
23+
let actual = actual.Value
2224
actual.Context.Collector.Errors |> shouldBeGreaterThan 0
2325
let errorDiagnostics = actual.Context.Collector.Diagnostics
2426
.Where(fun d -> d.Severity = Severity.Error)

‎tests/authoring/Framework/HtmlAssertions.fs‎

Lines changed: 18 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -89,35 +89,36 @@ actual: {actual}
8989
|> Seq.iter _.ToHtml(sw, PrettyMarkupFormatter())
9090
sw.ToString()
9191

92-
[<DebuggerStepThrough>]
93-
let convertsToHtml ([<LanguageInjection("html")>]expected: string) (actual: GenerateResult) =
92+
let private createDiff expected actual =
9493
let diffs =
9594
DiffBuilder
96-
.Compare(actual.Html)
95+
.Compare(actual)
9796
.WithTest(expected)
9897
.Build()
9998

100-
let diff = htmlDiffString diffs
101-
match diff with
99+
let deepComparision = htmlDiffString diffs
100+
match deepComparision with
102101
| s when String.IsNullOrEmpty s -> ()
103102
| s ->
104103
let expectedHtml = prettyHtml expected
105-
let actualHtml = prettyHtml actual.Html
106-
let textDiff =
107-
InlineDiffBuilder.Diff(expectedHtml, actualHtml).Lines
108-
|> Seq.map(fun l ->
109-
match l.Type with
110-
| ChangeType.Deleted -> "- " + l.Text
111-
| ChangeType.Modified -> "+ " + l.Text
112-
| ChangeType.Inserted -> "+ " + l.Text
113-
| _ -> " " + l.Text
114-
)
115-
|> String.concat "\n"
104+
let actualHtml = prettyHtml actual
105+
let textDiff = diff expectedHtml actualHtml
116106
let msg = $"""Html was not equal
117107
{textDiff}
118108
119-
{diff}
109+
{deepComparision}
120110
"""
121111
raise (XunitException(msg))
122112

113+
[<DebuggerStepThrough>]
114+
let convertsToHtml ([<LanguageInjection("html")>]expected: string) (actual: Lazy<GeneratorResults>) =
115+
let actual = actual.Value
116+
117+
let defaultFile = actual.MarkdownResults |> Seq.head
118+
createDiff expected defaultFile.Html
119+
120+
[<DebuggerStepThrough>]
121+
let toHtml ([<LanguageInjection("html")>]expected: string) (actual: MarkdownResult) =
122+
createDiff expected actual.Html
123+
123124

‎tests/authoring/Framework/MarkdownResultsAssertions.fs‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
namespace authoring
6+
7+
open System.Diagnostics
8+
open System.Text.Json
9+
open DiffPlex.DiffBuilder
10+
open DiffPlex.DiffBuilder.Model
11+
open FsUnit.Xunit
12+
open JetBrains.Annotations
13+
open Xunit.Sdk
14+
15+
[<AutoOpen>]
16+
module ResultsAssertions =
17+
18+
let diff expected actual =
19+
let diffLines = InlineDiffBuilder.Diff(expected, actual).Lines
20+
21+
let mutatedCount =
22+
diffLines
23+
|> Seq.filter (fun l ->
24+
match l.Type with
25+
| ChangeType.Modified -> true
26+
| ChangeType.Inserted -> true
27+
| _ -> false
28+
)
29+
|> Seq.length
30+
31+
let actualLineLength = actual.Split("\n").Length
32+
match mutatedCount with
33+
| 0 -> ""
34+
| _ when mutatedCount >= actualLineLength -> $"Mutations {mutatedCount} on all {actualLineLength} showing actual: \n\n{actual}"
35+
| _ ->
36+
diffLines
37+
|> Seq.map(fun l ->
38+
match l.Type with
39+
| ChangeType.Deleted -> "- " + l.Text
40+
| ChangeType.Modified -> "+ " + l.Text
41+
| ChangeType.Inserted -> "+ " + l.Text
42+
| _ -> " " + l.Text
43+
)
44+
|> String.concat "\n"
45+
46+
47+
[<DebuggerStepThrough>]
48+
let converts file (results: Lazy<GeneratorResults>) =
49+
let results = results.Value
50+
51+
let result =
52+
results.MarkdownResults
53+
|> Seq.tryFind (fun m -> m.File.RelativePath = file)
54+
55+
match result with
56+
| None ->
57+
raise (XunitException($"{file} not part of the markdown results"))
58+
| Some result -> result
59+
60+
[<AutoOpen>]
61+
module JsonAssertions =
62+
63+
[<DebuggerStepThrough>]
64+
let convertsToJson artifact ([<LanguageInjection("json")>]expected: string) (actual: Lazy<GeneratorResults>) =
65+
let actual = actual.Value
66+
let fs = actual.Context.ReadFileSystem
67+
68+
let fi = fs.FileInfo.New(artifact)
69+
if not <| fi.Exists then
70+
raise (XunitException($"{artifact} is not part of the output"))
71+
72+
let actual = fs.File.ReadAllText(fi.FullName)
73+
use actualJson = JsonDocument.Parse(actual);
74+
let actual = JsonSerializer.Serialize(actualJson, JsonSerializerOptions(WriteIndented = true))
75+
76+
use expectedJson = JsonDocument.Parse(expected);
77+
let expected = JsonSerializer.Serialize(expectedJson, JsonSerializerOptions(WriteIndented = true))
78+
79+
diff expected actual |> should be NullOrEmptyString
80+
81+

0 commit comments

Comments
 (0)