Merge branch 'main' of github.com:elastic/docs-builder into add-automated-guide

Author: bmorelli25

docs-builder.sln

@@ -51,6 +51,8 @@ Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "build", "build\build.fsproj
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "docs-assembler", "src\docs-assembler\docs-assembler.csproj", "{28350800-B44B-479B-86E2-1D39E321C0B4}"
 EndProject
+Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "authoring", "tests\authoring\authoring.fsproj", "{018F959E-824B-4664-B345-066784478D24}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -89,6 +91,10 @@ Global
 		{28350800-B44B-479B-86E2-1D39E321C0B4}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{28350800-B44B-479B-86E2-1D39E321C0B4}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{28350800-B44B-479B-86E2-1D39E321C0B4}.Release|Any CPU.Build.0 = Release|Any CPU
+		{018F959E-824B-4664-B345-066784478D24}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{018F959E-824B-4664-B345-066784478D24}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{018F959E-824B-4664-B345-066784478D24}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{018F959E-824B-4664-B345-066784478D24}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{4D198E25-C211-41DC-9E84-B15E89BD7048} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
@@ -99,5 +105,6 @@ Global
 		{CD2887E3-BDA9-434B-A5BF-9ED38DE20332} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
 		{A2A34BBC-CB5E-4100-9529-A12B6ECB769C} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
 		{28350800-B44B-479B-86E2-1D39E321C0B4} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
+		{018F959E-824B-4664-B345-066784478D24} = {67B576EE-02FA-4F9B-94BC-3630BC09ECE5}
 	EndGlobalSection
 EndGlobal

docs-builder.sln.DotSettings

@@ -1,4 +1,6 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=docset/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=frontmatter/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=linenos/@EntryIndexedValue">True</s:Boolean>
-	<s:Boolean x:Key="/Default/UserDictionary/Words/=literalinclude/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=literalinclude/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=_0060_0060inli/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

docs/migration/freeze/gh-action.md

@@ -27,6 +27,6 @@ jobs:
   comment-on-asciidoc-change:
     permissions:
       contents: read
-      pull_request: write
+      pull-requests: write
     uses: elastic/docs-builder/.github/workflows/comment-on-asciidoc-changes.yml@main
 ```

docs/testing/index.md

@@ -5,4 +5,29 @@ The files in this directory are used for testing purposes. Do not edit these fil
 
 ###### [#synthetics-config-file]
 
-% [Non Existing Link](./non-existing.md)
+% [Non Existing Link](./non-existing.md)
+
+```json
+{
+  "key": "value"
+}
+```
+
+  ```json
+  {
+    "key": "value"
+  }
+  ```
+
+1. this is a list
+   ```json
+      {
+        "key": "value"
+      }
+   ```
+   1. this is a sub-list
+      ```json
+      {
+        "key": "value"
+      }
+      ```

src/Elastic.Markdown/Helpers/Markdown.cs

@@ -0,0 +1,15 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+namespace Elastic.Markdown.Helpers;
+
+public static class MarkdownStringExtensions
+{
+	public static string StripMarkdown(this string markdown)
+	{
+		using var writer = new StringWriter();
+		Markdig.Markdown.ToPlainText(markdown, writer);
+		return writer.ToString().TrimEnd('\n');
+	}
+}

src/Elastic.Markdown/IO/MarkdownFile.cs

@@ -42,18 +42,28 @@ public DocumentationGroup? Parent
 	public string? UrlPathPrefix { get; }
 	private MarkdownParser MarkdownParser { get; }
 	public YamlFrontMatter? YamlFrontMatter { get; private set; }
-	public string? Title { get; private set; }
+	public string? TitleRaw { get; private set; }
+
+	public string? Title
+	{
+		get => _title;
+		private set
+		{
+			_title = value?.StripMarkdown();
+			TitleRaw = value;
+		}
+	}
 	public string? NavigationTitle
 	{
 		get => !string.IsNullOrEmpty(_navigationTitle) ? _navigationTitle : Title;
-		private set => _navigationTitle = value;
+		private set => _navigationTitle = value?.StripMarkdown();
 	}
 
 	//indexed by slug
-	private readonly Dictionary<string, PageTocItem> _tableOfContent = new();
+	private readonly Dictionary<string, PageTocItem> _tableOfContent = new(StringComparer.OrdinalIgnoreCase);
 	public IReadOnlyDictionary<string, PageTocItem> TableOfContents => _tableOfContent;
 
-	private readonly HashSet<string> _additionalLabels = new();
+	private readonly HashSet<string> _additionalLabels = new(StringComparer.OrdinalIgnoreCase);
 	public IReadOnlySet<string> AdditionalLabels => _additionalLabels;
 
 	public string FilePath { get; }
@@ -64,6 +74,7 @@ public string? NavigationTitle
 
 	private bool _instructionsParsed;
 	private DocumentationGroup? _parent;
+	private string? _title;
 
 	public MarkdownFile[] YieldParents()
 	{
@@ -156,7 +167,7 @@ private void ReadDocumentInstructions(MarkdownDocument document)
 			.Select(h => (h.GetData("header") as string, h.GetData("anchor") as string))
 			.Select(h => new PageTocItem
 			{
-				Heading = h.Item1!.Replace("`", "").Replace("*", ""),
+				Heading = h.Item1!.StripMarkdown(),
 				Slug = (h.Item2 ?? h.Item1).Slugify()
 			})
 			.ToList();

src/Elastic.Markdown/Myst/CodeBlocks/EnhancedCodeBlockHtmlRenderer.cs

@@ -5,6 +5,7 @@
 using Elastic.Markdown.Diagnostics;
 using Elastic.Markdown.Myst.Directives;
 using Elastic.Markdown.Slices.Directives;
+using Markdig.Helpers;
 using Markdig.Renderers;
 using Markdig.Renderers.Html;
 using Markdig.Syntax;
@@ -14,15 +15,85 @@ namespace Elastic.Markdown.Myst.CodeBlocks;
 
 public class EnhancedCodeBlockHtmlRenderer : HtmlObjectRenderer<EnhancedCodeBlock>
 {
+	private const int TabWidth = 4;
 
 	private static void RenderRazorSlice<T>(RazorSlice<T> slice, HtmlRenderer renderer, EnhancedCodeBlock block)
 	{
 		var html = slice.RenderAsync().GetAwaiter().GetResult();
 		var blocks = html.Split("[CONTENT]", 2, StringSplitOptions.RemoveEmptyEntries);
 		renderer.Write(blocks[0]);
-		renderer.WriteLeafRawLines(block, true, true, false);
+		RenderCodeBlockLines(renderer, block);
 		renderer.Write(blocks[1]);
 	}
+
+	/// <summary>
+	/// Renders the code block lines while also removing the common indentation level.
+	/// Required because EnableTrackTrivia preserves extra indentation.
+	/// </summary>
+	private static void RenderCodeBlockLines(HtmlRenderer renderer, EnhancedCodeBlock block)
+	{
+		var commonIndent = GetCommonIndent(block);
+		for (var i = 0; i < block.Lines.Count; i++)
+		{
+			var line = block.Lines.Lines[i];
+			var slice = line.Slice;
+			var indent = CountIndentation(slice);
+			if (indent >= commonIndent)
+				slice.Start += commonIndent;
+			RenderCodeBlockLine(renderer, block, slice, i);
+		}
+	}
+
+	private static void RenderCodeBlockLine(HtmlRenderer renderer, EnhancedCodeBlock block, StringSlice slice, int lineNumber)
+	{
+		renderer.WriteEscape(slice);
+		RenderCallouts(renderer, block, lineNumber);
+		renderer.WriteLine();
+	}
+
+	private static void RenderCallouts(HtmlRenderer renderer, EnhancedCodeBlock block, int lineNumber)
+	{
+		var callOuts = FindCallouts(block.CallOuts ?? [], lineNumber + 1);
+		foreach (var callOut in callOuts)
+			renderer.Write($"<span class=\"code-callout\">{callOut.Index}</span>");
+	}
+
+	private static IEnumerable<CallOut> FindCallouts(
+		IEnumerable<CallOut> callOuts,
+		int lineNumber
+	) => callOuts.Where(callOut => callOut.Line == lineNumber);
+
+	private static int GetCommonIndent(EnhancedCodeBlock block)
+	{
+		var commonIndent = int.MaxValue;
+		for (var i = 0; i < block.Lines.Count; i++)
+		{
+			var line = block.Lines.Lines[i].Slice;
+			if (line.IsEmptyOrWhitespace())
+				continue;
+			var indent = CountIndentation(line);
+			commonIndent = Math.Min(commonIndent, indent);
+		}
+		return commonIndent;
+	}
+
+
+	private static int CountIndentation(StringSlice slice)
+	{
+		var indentCount = 0;
+		for (var i = slice.Start; i <= slice.End; i++)
+		{
+			var c = slice.Text[i];
+			if (c == ' ')
+				indentCount++;
+			else if (c == '\t')
+				indentCount += TabWidth;
+			else
+				break;
+		}
+		return indentCount;
+	}
+
 	protected override void Write(HtmlRenderer renderer, EnhancedCodeBlock block)
 	{
 		var callOuts = block.UniqueCallOuts;

src/Elastic.Markdown/Myst/InlineParsers/HardBreakParser.cs

@@ -0,0 +1,65 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using Markdig;
+using Markdig.Helpers;
+using Markdig.Parsers;
+using Markdig.Parsers.Inlines;
+using Markdig.Renderers;
+using Markdig.Renderers.Html;
+using Markdig.Renderers.Html.Inlines;
+using Markdig.Syntax.Inlines;
+
+namespace Elastic.Markdown.Myst.InlineParsers;
+
+public static class HardBreakBuilderExtensions
+{
+	public static MarkdownPipelineBuilder UseHardBreaks(this MarkdownPipelineBuilder pipeline)
+	{
+		pipeline.Extensions.AddIfNotAlready<HardBreakBuilderExtension>();
+		return pipeline;
+	}
+}
+
+public class HardBreakBuilderExtension : IMarkdownExtension
+{
+	public void Setup(MarkdownPipelineBuilder pipeline) =>
+		pipeline.InlineParsers.InsertBefore<EmphasisInlineParser>(new HardBreakParser());
+
+	public void Setup(MarkdownPipeline pipeline, IMarkdownRenderer renderer) =>
+		renderer.ObjectRenderers.InsertAfter<EmphasisInlineRenderer>(new HardBreakRenderer());
+}
+
+public class HardBreakParser : InlineParser
+{
+	public HardBreakParser() => OpeningCharacters = ['<'];
+
+	public override bool Match(InlineProcessor processor, ref StringSlice slice)
+	{
+		var span = slice.AsSpan();
+		if (!span.StartsWith("<br"))
+			return false;
+
+		var closingStart = span[3..].IndexOf('>');
+		// we allow
+		if (closingStart != 0)
+			return false;
+
+		processor.Inline = new HardBreak();
+
+		var sliceEnd = slice.Start + 4; //<br + >
+		while (slice.Start != sliceEnd)
+			slice.SkipChar();
+
+		return true;
+	}
+}
+
+public class HardBreak : LeafInline;
+
+public class HardBreakRenderer : HtmlObjectRenderer<HardBreak>
+{
+	protected override void Write(HtmlRenderer renderer, HardBreak obj) =>
+		renderer.Write("<br>");
+}

src/Elastic.Markdown/Myst/MarkdownParser.cs

@@ -51,8 +51,10 @@ public class MarkdownParser(
 			.UseGridTables()
 			.UsePipeTables()
 			.UseDirectives()
+			.UseDefinitionLists()
 			.UseEnhancedCodeBlocks()
 			.DisableHtml()
+			.UseHardBreaks()
 			.Build();
 
 	public ConfigurationFile Configuration { get; } = configuration;
@@ -107,6 +109,4 @@ public MarkdownDocument Parse(string yaml, IFileInfo parent, YamlFrontMatter? ma
 		var markdownDocument = Markdig.Markdown.Parse(yaml, Pipeline, context);
 		return markdownDocument;
 	}
-
-
 }

src/Elastic.Markdown/Slices/HtmlWriter.cs

@@ -56,6 +56,7 @@ public async Task<string> RenderLayout(MarkdownFile markdown, Cancel ctx = defau
 		var slice = Index.Create(new IndexViewModel
 		{
 			Title = markdown.Title ?? "[TITLE NOT SET]",
+			TitleRaw = markdown.TitleRaw ?? "[TITLE NOT SET]",
 			MarkdownHtml = html,
 			PageTocItems = markdown.TableOfContents.Values.ToList(),
 			Tree = DocumentationSet.Tree,