Add global substitutions support to Markdown processing (#143)

Author: Mpdreamz

.github/workflows/pr.yml

@@ -15,13 +15,6 @@ env:
   NUGET_PACKAGES: ${{ github.workspace }}/.nuget/packages
 
 jobs:
-  docs:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Build documentation
-        uses: elastic/docs-builder@main
-  
   build:
     runs-on: ubuntu-latest
     steps:
@@ -39,4 +32,8 @@ jobs:
 
       - name: Publish AOT
         run: ./build.sh publishbinaries
-        
+
+      # we run our artifact directly please use the prebuild
+      # elastic/docs-builder@main GitHub Action for all other repositories!
+      - name: Build documentation
+        run: .artifacts/publish/docs-builder/release/docs-builder

docs/source/docset.yml

@@ -17,6 +17,8 @@ external_hosts:
   - palletsprojects.com
 exclude:
   - '_*.md'
+subs:
+  a-global-variable: "This was defined in docset.yml"
 toc:
   - file: index.md
   - folder: migration

docs/source/syntax/substitutions.md

@@ -6,12 +6,32 @@ sub:
   version: 7.17.0
 ---
 
+Substitutions can be defined in two places:
+
+1. In the `frontmatter` YAML within a file.
+2. Globally for all files in `docset.yml`
+
+In both cases the yaml to define them is as followed:
+
+
+```yaml
+subs:
+  key: value
+  another-var: Another Value
+```
+
+If a substitution is defined globally it may not be redefined (shaded) in a files `frontmatter`. 
+Doing so will result in a build error.
+
+## Example
+
 Here are some variable substitutions:
 
 | Variable              | Defined in   |
 |-----------------------|--------------|
 | {{frontmatter_key}}   | Front Matter |
 | {{a-key-with-dashes}} | Front Matter |
+| {{a-global-variable}} | `docset.yml` |
 
 Substitutions should work in code blocks too.
 

src/Elastic.Markdown/Diagnostics/ProcessorDiagnosticExtensions.cs

@@ -47,18 +47,15 @@ public static void EmitWarning(this InlineProcessor processor, int line, int col
 		context.Build.Collector.Channel.Write(d);
 	}
 
-	public static void EmitError(this ParserContext context, int line, int column, int length, string message, Exception? e = null)
+	public static void EmitError(this ParserContext context, string message, Exception? e = null)
 	{
 		if (context.SkipValidation)
 			return;
 		var d = new Diagnostic
 		{
 			Severity = Severity.Error,
 			File = context.Path.FullName,
-			Column = column,
-			Line = line,
 			Message = message + (e != null ? Environment.NewLine + e : string.Empty),
-			Length = length
 		};
 		context.Build.Collector.Channel.Write(d);
 	}

src/Elastic.Markdown/IO/ConfigurationFile.cs

@@ -29,6 +29,9 @@ public record ConfigurationFile : DocumentationFile
 		"github.com",
 	};
 
+	private readonly Dictionary<string, string> _substitutions = new(StringComparer.OrdinalIgnoreCase);
+	public IReadOnlyDictionary<string, string> Substitutions => _substitutions;
+
 	public ConfigurationFile(IFileInfo sourceFile, IDirectoryInfo rootPath, BuildContext context)
 		: base(sourceFile, rootPath)
 	{
@@ -70,6 +73,9 @@ public ConfigurationFile(IFileInfo sourceFile, IDirectoryInfo rootPath, BuildCon
 						.Select(Glob.Parse)
 						.ToArray();
 					break;
+				case "subs":
+					_substitutions = ReadDictionary(entry);
+					break;
 				case "external_hosts":
 					var hosts = ReadStringArray(entry)
 						.ToArray();
@@ -148,6 +154,27 @@ private List<ITocItem> ReadChildren(KeyValuePair<YamlNode, YamlNode> entry, stri
 		return null;
 	}
 
+	private Dictionary<string, string> ReadDictionary(KeyValuePair<YamlNode, YamlNode> entry)
+	{
+		var dictionary = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+		if (entry.Value is not YamlMappingNode mapping)
+		{
+			var key = ((YamlScalarNode)entry.Key).Value;
+			EmitWarning($"'{key}' is not a dictionary");
+			return dictionary;
+		}
+		foreach (var entryValue in mapping.Children)
+		{
+			if (entryValue.Key is not YamlScalarNode scalar || scalar.Value is null)
+				continue;
+			var key = scalar.Value;
+			var value = ReadString(entryValue);
+			if (value is not null)
+				dictionary.Add(key, value);
+		}
+		return dictionary;
+	}
+
 	private string? ReadFolder(KeyValuePair<YamlNode, YamlNode> entry, string parentPath, out bool found)
 	{
 		found = false;

src/Elastic.Markdown/Myst/Directives/IncludeBlock.cs

@@ -57,7 +57,7 @@ private void ExtractInclusionPath(ParserContext context)
 		var includePath = Arguments;
 		if (string.IsNullOrWhiteSpace(includePath))
 		{
-			context.EmitError(Line, Column, $"```{{{Directive}}}".Length, "include requires an argument.");
+			this.EmitError("include requires an argument.");
 			return;
 		}
 

src/Elastic.Markdown/Myst/ParserContext.cs

@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information
 
 using System.IO.Abstractions;
+using Elastic.Markdown.Diagnostics;
 using Elastic.Markdown.IO;
 using Elastic.Markdown.Myst.FrontMatter;
 using Markdig;
@@ -41,10 +42,19 @@ public ParserContext(MarkdownParser markdownParser,
 		Build = context;
 		Configuration = configuration;
 
+		foreach (var (key, value) in configuration.Substitutions)
+			Properties[key] = value;
+
 		if (frontMatter?.Properties is { } props)
 		{
 			foreach (var (key, value) in props)
-				Properties[key] = value;
+			{
+				if (configuration.Substitutions.TryGetValue(key, out _))
+					this.EmitError($"{{{key}}} can not be redeclared in front matter as its a global substitution");
+				else
+					Properties[key] = value;
+			}
+
 		}
 
 		if (frontMatter?.Title is { } title)

tests/Elastic.Markdown.Tests/Inline/InlneBaseTests.cs

@@ -78,7 +78,10 @@ public abstract class InlineTest : IAsyncLifetime
 	protected DocumentationSet Set { get; }
 
 
-	protected InlineTest(ITestOutputHelper output, [LanguageInjection("markdown")] string content)
+	protected InlineTest(
+		ITestOutputHelper output,
+		[LanguageInjection("markdown")] string content,
+		Dictionary<string, string>? globalVariables = null)
 	{
 		var logger = new TestLoggerFactory(output);
 		FileSystem = new MockFileSystem(new Dictionary<string, MockFileData>
@@ -102,7 +105,7 @@ protected InlineTest(ITestOutputHelper output, [LanguageInjection("markdown")] s
 		AddToFileSystem(FileSystem);
 
 		var root = FileSystem.DirectoryInfo.New(Path.Combine(Paths.Root.FullName, "docs/source"));
-		FileSystem.GenerateDocSetYaml(root);
+		FileSystem.GenerateDocSetYaml(root, globalVariables);
 
 		Collector = new TestDiagnosticsCollector(logger);
 		var context = new BuildContext(FileSystem)

tests/Elastic.Markdown.Tests/Inline/SubstitutionTest.cs

@@ -22,7 +22,7 @@ not a comment
 {
 
 	[Fact]
-	public void GeneratesAttributesInHtml() =>
+	public void ReplacesSubsFromFrontMatter() =>
 		Html.Should().Contain(
 				"""Hello World!<br />"""
 			).And.Contain(
@@ -48,7 +48,7 @@ not a {substitution}
 {
 
 	[Fact]
-	public void GeneratesAttributesInHtml() =>
+	public void PreservesSingleBracket() =>
 		Html.Should().Contain(
 				"""Hello World!<br />"""
 			).And.Contain(
@@ -87,3 +87,49 @@ public void ReplacesSubsInCode() =>
 		Html.Should().Contain("7.17.0");
 }
 
+
+public class SupportsSubstitutionsFromDocSet(ITestOutputHelper output) : InlineTest(output,
+"""
+---
+sub:
+  hello-world: "Hello World!"
+---
+The following should be subbed: {{hello-world}}
+The following should be subbed as well: {{global-var}}
+"""
+, new() { { "global-var", "A variable from docset.yml" } }
+)
+{
+
+	[Fact]
+	public void EmitsGlobalVariable() =>
+		Html.Should().Contain("Hello World!<br />")
+			.And.NotContain("{{hello-world}}")
+			.And.Contain("A variable from docset.yml")
+			.And.NotContain("{{global-var}}");
+}
+
+
+public class CanNotShadeGlobalVariables(ITestOutputHelper output) : InlineTest(output,
+"""
+---
+sub:
+  hello-world: "Hello World!"
+---
+The following should be subbed: {{hello-world}}
+The following should be subbed as well: {{hello-world}}
+"""
+, new() { { "hello-world", "A variable from docset.yml" } }
+)
+{
+
+	[Fact]
+	public void OnlySeesGlobalVariable() =>
+		Html.Should().NotContain("Hello World!<br />")
+			.And.NotContain("{{hello-world}}")
+			.And.Contain("A variable from docset.yml");
+
+	[Fact]
+	public void HasError() => Collector.Diagnostics.Should().HaveCount(1)
+		.And.Contain(d => d.Message.Contains("{hello-world} can not be redeclared in front matter as its a global substitution"));
+}

tests/Elastic.Markdown.Tests/MockFileSystemExtensions.cs

@@ -9,7 +9,7 @@ namespace Elastic.Markdown.Tests;
 
 public static class MockFileSystemExtensions
 {
-	public static void GenerateDocSetYaml(this MockFileSystem fileSystem, IDirectoryInfo root)
+	public static void GenerateDocSetYaml(this MockFileSystem fileSystem, IDirectoryInfo root, Dictionary<string, string>? globalVariables = null)
 	{
 		// language=yaml
 		var yaml = new StringWriter();
@@ -21,6 +21,14 @@ public static void GenerateDocSetYaml(this MockFileSystem fileSystem, IDirectory
 			var relative = fileSystem.Path.GetRelativePath(root.FullName, markdownFile);
 			yaml.WriteLine($" - file: {relative}");
 		}
+
+		if (globalVariables is not null)
+		{
+			yaml.WriteLine($"subs:");
+			foreach (var (key, value) in globalVariables)
+				yaml.WriteLine($"  {key}: {value}");
+		}
+
 		fileSystem.AddFile(Path.Combine(root.FullName, "docset.yml"), new MockFileData(yaml.ToString()));
 	}