Skip to content

Commit ea43fc2

Browse files
MpdreamzMpdreamz
authored
New test project to make specification tests easier to write for developers (#344)
* Spike new test project to improve test readability This is a spike to make it easier to write markdown specification tests. It allows us to test markdown fragments and their resulting HTML much terser while making it easier to come up with good test names. * move to xunit v3 and unquote * license headers * add github logger * remove BOM
1 parent 69fdd21 commit ea43fc2

File tree

12 files changed

+477
-2
lines changed

12 files changed

+477
-2
lines changed

‎docs-builder.sln‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "build", "build\build.fsproj
5151
EndProject
5252
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "docs-assembler", "src\docs-assembler\docs-assembler.csproj", "{28350800-B44B-479B-86E2-1D39E321C0B4}"
5353
EndProject
54+
Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "authoring", "tests\authoring\authoring.fsproj", "{018F959E-824B-4664-B345-066784478D24}"
55+
EndProject
5456
Global
5557
GlobalSection(SolutionConfigurationPlatforms) = preSolution
5658
Debug|Any CPU = Debug|Any CPU
@@ -89,6 +91,10 @@ Global
8991
{28350800-B44B-479B-86E2-1D39E321C0B4}.Debug|Any CPU.Build.0 = Debug|Any CPU
9092
{28350800-B44B-479B-86E2-1D39E321C0B4}.Release|Any CPU.ActiveCfg = Release|Any CPU
9193
{28350800-B44B-479B-86E2-1D39E321C0B4}.Release|Any CPU.Build.0 = Release|Any CPU
94+
{018F959E-824B-4664-B345-066784478D24}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
95+
{018F959E-824B-4664-B345-066784478D24}.Debug|Any CPU.Build.0 = Debug|Any CPU
96+
{018F959E-824B-4664-B345-066784478D24}.Release|Any CPU.ActiveCfg = Release|Any CPU
97+
{018F959E-824B-4664-B345-066784478D24}.Release|Any CPU.Build.0 = Release|Any CPU
9298
EndGlobalSection
9399
GlobalSection(NestedProjects) = preSolution
94100
{4D198E25-C211-41DC-9E84-B15E89BD7048} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
@@ -99,5 +105,6 @@ Global
99105
{CD2887E3-BDA9-434B-A5BF-9ED38DE20332} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
100106
{A2A34BBC-CB5E-4100-9529-A12B6ECB769C} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
101107
{28350800-B44B-479B-86E2-1D39E321C0B4} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
108+
{018F959E-824B-4664-B345-066784478D24} = {67B576EE-02FA-4F9B-94BC-3630BC09ECE5}
102109
EndGlobalSection
103110
EndGlobal

‎docs-builder.sln.DotSettings‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
22
<s:Boolean x:Key="/Default/UserDictionary/Words/=docset/@EntryIndexedValue">True</s:Boolean>
3+
<s:Boolean x:Key="/Default/UserDictionary/Words/=frontmatter/@EntryIndexedValue">True</s:Boolean>
34
<s:Boolean x:Key="/Default/UserDictionary/Words/=linenos/@EntryIndexedValue">True</s:Boolean>
4-
<s:Boolean x:Key="/Default/UserDictionary/Words/=literalinclude/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>
5+
<s:Boolean x:Key="/Default/UserDictionary/Words/=literalinclude/@EntryIndexedValue">True</s:Boolean>
6+
<s:Boolean x:Key="/Default/UserDictionary/Words/=_0060_0060inli/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

‎tests/Elastic.Markdown.Tests/Inline/DirectiveBlockLinkTests.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ protected override void AddToFileSystem(MockFileSystem fileSystem)
3131
3232
:::{important}
3333
:name: hint_ref
34-
This is a 'important' admonition
34+
This is an 'important' admonition
3535
:::
3636
""";
3737
fileSystem.AddFile(@"docs/testing/req.md", inclusion);

‎tests/authoring/Framework/ErrorCollectorAssertions.fs‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
namespace authoring
6+
7+
open System.Diagnostics
8+
open System.Linq
9+
open Elastic.Markdown.Diagnostics
10+
open FsUnitTyped
11+
open Swensen.Unquote
12+
13+
[<AutoOpen>]
14+
module DiagnosticsCollectorAssertions =
15+
16+
[<DebuggerStepThrough>]
17+
let hasNoErrors (actual: GenerateResult) =
18+
test <@ actual.Context.Collector.Errors = 0 @>
19+
20+
[<DebuggerStepThrough>]
21+
let hasError (expected: string) (actual: GenerateResult) =
22+
actual.Context.Collector.Errors |> shouldBeGreaterThan 0
23+
let errorDiagnostics = actual.Context.Collector.Diagnostics
24+
.Where(fun d -> d.Severity = Severity.Error)
25+
.ToArray()
26+
|> List.ofArray
27+
let message = errorDiagnostics.FirstOrDefault().Message
28+
test <@ message.Contains(expected) @>

‎tests/authoring/Framework/HtmlAssertions.fs‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
namespace authoring
6+
7+
open System
8+
open System.Diagnostics
9+
open System.IO
10+
open AngleSharp.Diffing
11+
open AngleSharp.Diffing.Core
12+
open AngleSharp.Html
13+
open AngleSharp.Html.Parser
14+
open DiffPlex.DiffBuilder
15+
open DiffPlex.DiffBuilder.Model
16+
open JetBrains.Annotations
17+
open Xunit.Sdk
18+
19+
[<AutoOpen>]
20+
module HtmlAssertions =
21+
22+
let htmlDiffString (diffs: seq<IDiff>) =
23+
let NodeName (source:ComparisonSource) = source.Node.NodeType.ToString().ToLowerInvariant();
24+
let htmlText (source:IDiff) =
25+
let formatter = PrettyMarkupFormatter();
26+
let nodeText (control: ComparisonSource) =
27+
use sw = new StringWriter()
28+
control.Node.ToHtml(sw, formatter)
29+
sw.ToString()
30+
let attrText (control: AttributeComparisonSource) =
31+
use sw = new StringWriter()
32+
control.Attribute.ToHtml(sw, formatter)
33+
sw.ToString()
34+
let nodeDiffText (control: ComparisonSource option) (test: ComparisonSource option) =
35+
let actual = match test with Some t -> nodeText t | None -> "missing"
36+
let expected = match control with Some t -> nodeText t | None -> "missing"
37+
$"""
38+
39+
expected: {expected}
40+
actual: {actual}
41+
"""
42+
let attrDiffText (control: AttributeComparisonSource option) (test: AttributeComparisonSource option) =
43+
let actual = match test with Some t -> attrText t | None -> "missing"
44+
let expected = match control with Some t -> attrText t | None -> "missing"
45+
$"""
46+
47+
expected: {expected}
48+
actual: {actual}
49+
"""
50+
51+
match source with
52+
| :? NodeDiff as diff -> nodeDiffText <| Some diff.Control <| Some diff.Test
53+
| :? AttrDiff as diff -> attrDiffText <| Some diff.Control <| Some diff.Test
54+
| :? MissingNodeDiff as diff -> nodeDiffText <| Some diff.Control <| None
55+
| :? MissingAttrDiff as diff -> attrDiffText <| Some diff.Control <| None
56+
| :? UnexpectedNodeDiff as diff -> nodeDiffText None <| Some diff.Test
57+
| :? UnexpectedAttrDiff as diff -> attrDiffText None <| Some diff.Test
58+
| _ -> failwith $"Unknown diff type detected: {source.GetType()}"
59+
60+
diffs
61+
|> Seq.map (fun diff ->
62+
63+
match diff with
64+
| :? NodeDiff as diff when diff.Target = DiffTarget.Text && diff.Control.Path.Equals(diff.Test.Path, StringComparison.Ordinal)
65+
-> $"The text in {diff.Control.Path} is different."
66+
| :? NodeDiff as diff when diff.Target = DiffTarget.Text
67+
-> $"The expected {NodeName(diff.Control)} at {diff.Control.Path} and the actual {NodeName(diff.Test)} at {diff.Test.Path} is different."
68+
| :? NodeDiff as diff when diff.Control.Path.Equals(diff.Test.Path, StringComparison.Ordinal)
69+
-> $"The {NodeName(diff.Control)}s at {diff.Control.Path} are different."
70+
| :? NodeDiff as diff -> $"The expected {NodeName(diff.Control)} at {diff.Control.Path} and the actual {NodeName(diff.Test)} at {diff.Test.Path} are different."
71+
| :? AttrDiff as diff when diff.Control.Path.Equals(diff.Test.Path, StringComparison.Ordinal)
72+
-> $"The values of the attributes at {diff.Control.Path} are different."
73+
| :? AttrDiff as diff -> $"The value of the attribute {diff.Control.Path} and actual attribute {diff.Test.Path} are different."
74+
| :? MissingNodeDiff as diff -> $"The {NodeName(diff.Control)} at {diff.Control.Path} is missing."
75+
| :? MissingAttrDiff as diff -> $"The attribute at {diff.Control.Path} is missing."
76+
| :? UnexpectedNodeDiff as diff -> $"The {NodeName(diff.Test)} at {diff.Test.Path} was not expected."
77+
| :? UnexpectedAttrDiff as diff -> $"The attribute at {diff.Test.Path} was not expected."
78+
| _ -> failwith $"Unknown diff type detected: {diff.GetType()}"
79+
+
80+
htmlText diff
81+
)
82+
|> String.concat "\n"
83+
84+
let private prettyHtml (html:string) =
85+
let parser = HtmlParser()
86+
let document = parser.ParseDocument(html)
87+
use sw = new StringWriter()
88+
document.Body.Children
89+
|> Seq.iter _.ToHtml(sw, PrettyMarkupFormatter())
90+
sw.ToString()
91+
92+
[<DebuggerStepThrough>]
93+
let convertsToHtml ([<LanguageInjection("html")>]expected: string) (actual: GenerateResult) =
94+
let diffs =
95+
DiffBuilder
96+
.Compare(actual.Html)
97+
.WithTest(expected)
98+
.Build()
99+
100+
let diff = htmlDiffString diffs
101+
match diff with
102+
| s when String.IsNullOrEmpty s -> ()
103+
| s ->
104+
let expectedHtml = prettyHtml expected
105+
let actualHtml = prettyHtml actual.Html
106+
let textDiff =
107+
InlineDiffBuilder.Diff(expectedHtml, actualHtml).Lines
108+
|> Seq.map(fun l ->
109+
match l.Type with
110+
| ChangeType.Deleted -> "- " + l.Text
111+
| ChangeType.Modified -> "+ " + l.Text
112+
| ChangeType.Inserted -> "+ " + l.Text
113+
| _ -> " " + l.Text
114+
)
115+
|> String.concat "\n"
116+
let msg = $"""Html was not equal
117+
{textDiff}
118+
119+
{diff}
120+
"""
121+
raise (XunitException(msg))
122+
123+

‎tests/authoring/Framework/Setup.fs‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
namespace authoring
6+
7+
open System.Collections.Generic
8+
open System.IO
9+
open System.IO.Abstractions.TestingHelpers
10+
open System.Threading.Tasks
11+
open Elastic.Markdown
12+
open Elastic.Markdown.IO
13+
open JetBrains.Annotations
14+
15+
type Setup =
16+
17+
static let GenerateDocSetYaml(
18+
fileSystem: MockFileSystem,
19+
globalVariables: Dictionary<string, string> option
20+
) =
21+
let root = fileSystem.DirectoryInfo.New(Path.Combine(Paths.Root.FullName, "docs/"));
22+
let yaml = new StringWriter();
23+
yaml.WriteLine("toc:");
24+
let markdownFiles = fileSystem.Directory.EnumerateFiles(root.FullName, "*.md", SearchOption.AllDirectories)
25+
markdownFiles
26+
|> Seq.iter(fun markdownFile ->
27+
let relative = fileSystem.Path.GetRelativePath(root.FullName, markdownFile);
28+
yaml.WriteLine($" - file: {relative}");
29+
)
30+
match globalVariables with
31+
| Some vars ->
32+
yaml.WriteLine($"subs:")
33+
vars |> Seq.iter(fun kv ->
34+
yaml.WriteLine($" {kv.Key}: {kv.Value}");
35+
)
36+
| _ -> ()
37+
38+
fileSystem.AddFile(Path.Combine(root.FullName, "docset.yml"), MockFileData(yaml.ToString()));
39+
40+
static let Generate ([<LanguageInjection("markdown")>]m: string) : Task<GenerateResult> =
41+
42+
let d = dict [ ("docs/index.md", MockFileData(m)) ]
43+
let opts = MockFileSystemOptions(CurrentDirectory=Paths.Root.FullName)
44+
let fileSystem = MockFileSystem(d, opts)
45+
46+
GenerateDocSetYaml (fileSystem, None)
47+
48+
let collector = TestDiagnosticsCollector();
49+
let context = BuildContext(fileSystem, Collector=collector)
50+
let set = DocumentationSet(context);
51+
let file =
52+
match set.GetMarkdownFile(fileSystem.FileInfo.New("docs/index.md")) with
53+
| NonNull f -> f
54+
| _ -> failwithf "docs/index.md could not be located"
55+
56+
let context = {
57+
File = file
58+
Collector = collector
59+
Set = set
60+
ReadFileSystem = fileSystem
61+
WriteFileSystem = fileSystem
62+
}
63+
context.Bootstrap()
64+
65+
/// Pass a full documentation page to the test setup
66+
static member Document ([<LanguageInjection("markdown")>]m: string) =
67+
let g = task { return! Generate m }
68+
g |> Async.AwaitTask |> Async.RunSynchronously
69+
70+
/// Pass a markdown fragment to the test setup
71+
static member Markdown ([<LanguageInjection("markdown")>]m: string) =
72+
// language=markdown
73+
let m = $"""
74+
# Test Document
75+
{m}
76+
"""
77+
let g = task {
78+
return! Generate m
79+
}
80+
g |> Async.AwaitTask |> Async.RunSynchronously
81+

‎tests/authoring/Framework/TestValues.fs‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
namespace authoring
6+
7+
open System
8+
open System.IO.Abstractions
9+
open Elastic.Markdown.Diagnostics
10+
open Elastic.Markdown.IO
11+
open Markdig.Syntax
12+
open Xunit
13+
14+
15+
type TestDiagnosticsOutput() =
16+
17+
interface IDiagnosticsOutput with
18+
member this.Write diagnostic =
19+
let line = match diagnostic.Line with | NonNullV l -> l | _ -> 0
20+
match TestContext.Current.TestOutputHelper with
21+
| NonNull output ->
22+
match diagnostic.Severity with
23+
| Severity.Error ->
24+
output.WriteLine($"Error: {diagnostic.Message} ({diagnostic.File}:{line})")
25+
| _ ->
26+
output.WriteLine($"Warn : {diagnostic.Message} ({diagnostic.File}:{line})")
27+
| _ -> ()
28+
29+
30+
type TestDiagnosticsCollector() =
31+
inherit DiagnosticsCollector([TestDiagnosticsOutput()])
32+
33+
let diagnostics = System.Collections.Generic.List<Diagnostic>()
34+
35+
member _.Diagnostics = diagnostics.AsReadOnly()
36+
37+
override this.HandleItem diagnostic = diagnostics.Add(diagnostic);
38+
39+
type GenerateResult = {
40+
Document: MarkdownDocument
41+
Html: string
42+
Context: MarkdownTestContext
43+
}
44+
45+
and MarkdownTestContext =
46+
{
47+
File: MarkdownFile
48+
Collector: TestDiagnosticsCollector
49+
Set: DocumentationSet
50+
ReadFileSystem: IFileSystem
51+
WriteFileSystem: IFileSystem
52+
}
53+
54+
member this.Bootstrap () = backgroundTask {
55+
let! ctx = Async.CancellationToken
56+
let _ = this.Collector.StartAsync(ctx)
57+
do! this.Set.ResolveDirectoryTree(ctx)
58+
59+
let! document = this.File.ParseFullAsync(ctx)
60+
61+
let html = this.File.CreateHtml(document);
62+
this.Collector.Channel.TryComplete()
63+
do! this.Collector.StopAsync(ctx)
64+
return { Context = this; Document = document; Html = html }
65+
}
66+
67+
interface IDisposable with
68+
member this.Dispose() = ()
69+
70+

‎tests/authoring/Inline/Comments.fs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
module ``inline elements``.``comment block``
6+
7+
open Xunit
8+
open authoring
9+
10+
type ``commented line`` () =
11+
12+
static let markdown = Setup.Markdown """
13+
% comment
14+
not a comment
15+
"""
16+
17+
[<Fact>]
18+
let ``validate HTML: commented line should not be emitted`` () =
19+
markdown |> convertsToHtml """<p>not a comment</p>"""

‎tests/authoring/Inline/InlineAnchors.fs‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
module ``inline elements``.``anchors DEPRECATED``
6+
7+
open Xunit
8+
open authoring
9+
10+
type ``inline anchor in the middle`` () =
11+
12+
static let markdown = Setup.Markdown """
13+
this is *regular* text and this $$$is-an-inline-anchor$$$ and this continues to be regular text
14+
"""
15+
16+
[<Fact>]
17+
let ``validate HTML`` () =
18+
markdown |> convertsToHtml """
19+
<p>this is <em>regular</em> text and this
20+
<a id="is-an-inline-anchor"></a> and this continues to be regular text
21+
</p>
22+
"""
23+
[<Fact>]
24+
let ``has no errors`` () = markdown |> hasNoErrors

0 commit comments

Comments
 (0)