Add extension hook points to all special usecases to inject themselves into docs build process (#666)

* Move detection-rules support behind an extension system

* Move EnabledExtensions to Configuration

* add untracked files

* Add documentation for extensions

* Add more extensions points to isolate the DetectionRule extension further

* Isolate file scanning in DocumentationSet

* Do a ful parse

Author: Mpdreamz

.editorconfig

@@ -229,13 +229,15 @@ dotnet_diagnostic.IDE0057.severity = none
 dotnet_diagnostic.IDE0051.severity = suggestion
 dotnet_diagnostic.IDE0059.severity = suggestion
 
+dotnet_diagnostic.CA1859.severity = none
+
+dotnet_diagnostic.IDE0305.severity = none
+
 
 [DocumentationWebHost.cs]
 dotnet_diagnostic.IL3050.severity = none
 dotnet_diagnostic.IL2026.severity = none
 
-
-
 [tests/**/*.cs]
 dotnet_diagnostic.IDE0058.severity = none
 dotnet_diagnostic.IDE0022.severity = none

docs/_docset.yml

@@ -58,6 +58,7 @@ toc:
           - file: file-structure.md
           - file: attributes.md
           - file: navigation.md
+          - file: extensions.md
       - file: page.md
   - folder: syntax
     children:

docs/configure/content-set/extensions.md

@@ -0,0 +1,32 @@
+---
+navigation_title: Extensions
+---
+
+# Content set extensions. 
+
+The documentation engineering team will on occasion built extensions for specific use-cases.
+
+These extension needs to be explicitly opted into since they typically only apply to a few content sets.
+
+
+## Detection Rules Extensions
+
+For the TRADE team the team built in support to picking up detection rule files from the source and emitting 
+documentation and navigation for them.
+
+To enable:
+
+```yaml
+extensions:
+  - detection-rules
+```
+
+This now allows you to use the special `detection_rules` instruction in the [Table of Contents](navigation.md)
+As a means to pick up `toml` files as `children`
+
+```yaml
+toc:
+  - file: index.md
+    detection_rules: '../rules'
+```
+

src/Elastic.Markdown/BuildContext.cs

@@ -4,6 +4,7 @@
 
 using System.IO.Abstractions;
 using Elastic.Markdown.Diagnostics;
+using Elastic.Markdown.Extensions;
 using Elastic.Markdown.IO;
 using Elastic.Markdown.IO.Configuration;
 using Elastic.Markdown.IO.Discovery;
@@ -18,6 +19,8 @@ public record BuildContext
 	public IDirectoryInfo DocumentationSourceDirectory { get; }
 	public IDirectoryInfo DocumentationOutputDirectory { get; }
 
+	public ConfigurationFile Configuration { get; set; }
+
 	public IFileInfo ConfigurationPath { get; }
 
 	public GitCheckoutInformation Git { get; }
@@ -67,9 +70,8 @@ public BuildContext(DiagnosticsCollector collector, IFileSystem readFileSystem,
 
 		Git = GitCheckoutInformation.Create(DocumentationSourceDirectory, ReadFileSystem);
 		Configuration = new ConfigurationFile(ConfigurationPath, DocumentationSourceDirectory, this);
-	}
 
-	public ConfigurationFile Configuration { get; set; }
+	}
 
 	private (IDirectoryInfo, IFileInfo) FindDocsFolderFromRoot(IDirectoryInfo rootPath)
 	{
@@ -95,4 +97,5 @@ from folder in knownFolders
 
 		return (docsFolder, configurationPath);
 	}
+
 }

src/Elastic.Markdown/Elastic.Markdown.csproj

@@ -51,6 +51,7 @@
 
   <ItemGroup>
     <PackageReference Include="DotNet.Glob" Version="3.1.3" />
+    <PackageReference Include="Samboy063.Tomlet" Version="6.0.0" />
     <PackageReference Include="SoftCircuits.IniFileParser" Version="2.6.0" />
     <PackageReference Include="Markdig" Version="0.39.1" />
     <PackageReference Include="Microsoft.Extensions.Logging" Version="9.0.0" />

src/Elastic.Markdown/Extensions/DetectionRules/DetectionRule.cs

@@ -0,0 +1,95 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Diagnostics.CodeAnalysis;
+using System.IO.Abstractions;
+using Tomlet;
+using Tomlet.Models;
+
+namespace Elastic.Markdown.Extensions.DetectionRules;
+
+public record DetectionRule
+{
+	public required string Name { get; init; }
+
+	public required string[]? Authors { get; init; }
+
+	public required string? Note { get; init; }
+
+	public required string? Query { get; init; }
+
+	public required string[]? Tags { get; init; }
+
+	public required string Severity { get; init; }
+
+	public required string RuleId { get; init; }
+
+	public required int RiskScore { get; init; }
+
+	public required string License { get; init; }
+
+	public required string Description { get; init; }
+	public required string Type { get; init; }
+	public required string? Language { get; init; }
+	public required string[]? Indices { get; init; }
+	public required string RunsEvery { get; init; }
+	public required string? IndicesFromDateMath { get; init; }
+	public required string MaximumAlertsPerExecution { get; init; }
+	public required string[]? References { get; init; }
+	public required string Version { get; init; }
+
+	public static DetectionRule From(IFileInfo source)
+	{
+		TomlDocument model;
+		try
+		{
+			var sourceText = File.ReadAllText(source.FullName);
+			model = new TomlParser().Parse(sourceText);
+		}
+		catch (Exception e)
+		{
+			throw new Exception($"Could not parse toml in: {source.FullName}", e);
+		}
+		if (!model.TryGetValue("metadata", out var node) || node is not TomlTable metadata)
+			throw new Exception($"Could not find metadata section in {source.FullName}");
+
+		if (!model.TryGetValue("rule", out node) || node is not TomlTable rule)
+			throw new Exception($"Could not find rule section in {source.FullName}");
+
+		return new DetectionRule
+		{
+			Authors = TryGetStringArray(rule, "author"),
+			Description = rule.GetString("description"),
+			Type = rule.GetString("type"),
+			Language = TryGetString(rule, "language"),
+			License = rule.GetString("license"),
+			RiskScore = TryRead<int>(rule, "risk_score"),
+			RuleId = rule.GetString("rule_id"),
+			Severity = rule.GetString("severity"),
+			Tags = TryGetStringArray(rule, "tags"),
+			Indices = TryGetStringArray(rule, "index"),
+			References = TryGetStringArray(rule, "references"),
+			IndicesFromDateMath = TryGetString(rule, "from"),
+			Query = TryGetString(rule, "query"),
+			Note = TryGetString(rule, "note"),
+			Name = rule.GetString("name"),
+			RunsEvery = "?",
+			MaximumAlertsPerExecution = "?",
+			Version = "?",
+		};
+	}
+
+	private static string[]? TryGetStringArray(TomlTable table, string key) =>
+		table.TryGetValue(key, out var node) && node is TomlArray t ? t.ArrayValues.Select(value => value.StringValue).ToArray() : null;
+
+	private static string? TryGetString(TomlTable table, string key) =>
+		table.TryGetValue(key, out var node) && node is TomlString t ? t.Value : null;
+
+	private static TTarget? TryRead<TTarget>(TomlTable table, string key) =>
+		table.TryGetValue(key, out var node) && node is TTarget t ? t : default;
+
+	private static TTarget Read<TTarget>(TomlTable table, string key) =>
+		TryRead<TTarget>(table, key) ?? throw new Exception($"Could not find {key} in {table}");
+
+}

src/Elastic.Markdown/Extensions/DetectionRules/DetectionRuleFile.cs

@@ -0,0 +1,77 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.IO.Abstractions;
+using Elastic.Markdown.IO;
+using Elastic.Markdown.Myst;
+using Markdig.Syntax;
+
+namespace Elastic.Markdown.Extensions.DetectionRules;
+
+public record DetectionRuleFile : MarkdownFile
+{
+	public DetectionRule? Rule { get; set; }
+
+	public DetectionRuleFile(
+		IFileInfo sourceFile,
+		IDirectoryInfo rootPath,
+		MarkdownParser parser,
+		BuildContext build,
+		DocumentationSet set
+	) : base(sourceFile, rootPath, parser, build, set)
+	{
+	}
+
+	protected override string RelativePathUrl => RelativePath.AsSpan().TrimStart("../").ToString();
+
+	protected override Task<MarkdownDocument> GetMinimalParseDocumentAsync(Cancel ctx)
+	{
+		var document = MarkdownParser.MinimalParseStringAsync(Rule?.Note ?? string.Empty, SourceFile, null);
+		Title = Rule?.Name;
+		return Task.FromResult(document);
+	}
+
+	protected override Task<MarkdownDocument> GetParseDocumentAsync(Cancel ctx)
+	{
+		if (Rule == null)
+			return Task.FromResult(MarkdownParser.ParseStringAsync($"# {Title}", SourceFile, null));
+
+		// language=markdown
+		var markdown = $"""
+# {Rule.Name}
+
+**Rule type**: {Rule.Type}
+**Rule indices**: {RenderArray(Rule.Indices)}
+**Rule Severity**: {Rule.Severity}
+**Risk Score**: {Rule.RiskScore}
+**Runs every**: {Rule.RunsEvery}
+**Searches indices from**: `{Rule.IndicesFromDateMath}`
+**Maximum alerts per execution**: {Rule.MaximumAlertsPerExecution}
+**References**: {RenderArray(Rule.References)}
+**Tags**: {RenderArray(Rule.Tags)}
+**Version**: {Rule.Version}
+**Rule authors**: {RenderArray(Rule.Authors)}
+**Rule license**: {Rule.License}
+
+## Investigation guide
+
+{Rule.Note}
+
+## Rule Query
+
+```{Rule.Language ?? Rule.Type}
+{Rule.Query}
+```
+""";
+		var document = MarkdownParser.ParseStringAsync(markdown, SourceFile, null);
+		return Task.FromResult(document);
+	}
+
+	private static string RenderArray(string[]? values)
+	{
+		if (values == null || values.Length == 0)
+			return string.Empty;
+		return "\n - " + string.Join("\n - ", values) + "\n";
+	}
+}

src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesDocsBuilderExtension.cs

@@ -0,0 +1,111 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.IO.Abstractions;
+using Elastic.Markdown.IO;
+using Elastic.Markdown.IO.Configuration;
+using Elastic.Markdown.IO.Navigation;
+
+namespace Elastic.Markdown.Extensions.DetectionRules;
+
+public class DetectionRulesDocsBuilderExtension(BuildContext build) : IDocsBuilderExtension
+{
+	private BuildContext Build { get; } = build;
+	public bool InjectsIntoNavigation(ITocItem tocItem) => tocItem is RulesFolderReference;
+
+	public void CreateNavigationItem(
+		DocumentationGroup? parent,
+		ITocItem tocItem,
+		NavigationLookups lookups,
+		List<DocumentationGroup> groups,
+		List<INavigationItem> navigationItems,
+		int depth,
+		ref int fileIndex,
+		int index)
+	{
+		var detectionRulesFolder = (RulesFolderReference)tocItem;
+		var children = detectionRulesFolder.Children;
+		var group = new DocumentationGroup(Build, lookups with { TableOfContents = children }, ref fileIndex, depth + 1)
+		{
+			Parent = parent
+		};
+		groups.Add(group);
+		navigationItems.Add(new GroupNavigation(index, depth, group));
+	}
+
+	public void Visit(DocumentationFile file, ITocItem tocItem)
+	{
+		// ensure the file has an instance of the rule the reference parsed.
+		if (file is DetectionRuleFile df && tocItem is RuleReference r)
+			df.Rule = r.Rule;
+	}
+
+	public DocumentationFile? CreateDocumentationFile(IFileInfo file, IDirectoryInfo sourceDirectory, DocumentationSet documentationSet)
+	{
+		if (file.Extension != ".toml")
+			return null;
+
+		return new DetectionRuleFile(file, Build.DocumentationSourceDirectory, documentationSet.MarkdownParser, Build, documentationSet);
+	}
+
+	public bool TryGetDocumentationFileBySlug(DocumentationSet documentationSet, string slug, out DocumentationFile? documentationFile)
+	{
+		var tomlFile = $"../{slug}.toml";
+		return documentationSet.FlatMappedFiles.TryGetValue(tomlFile, out documentationFile);
+	}
+
+	public IReadOnlyCollection<DocumentationFile> ScanDocumentationFiles(
+		Func<BuildContext, IDirectoryInfo, DocumentationFile[]> scanDocumentationFiles,
+		Func<IFileInfo, IDirectoryInfo, DocumentationFile> defaultFileHandling
+	)
+	{
+		var rules = Build.Configuration.TableOfContents.OfType<FileReference>().First().Children.OfType<RuleReference>().ToArray();
+		if (rules.Length == 0)
+			return [];
+
+		var sourcePath = Path.GetFullPath(Path.Combine(Build.DocumentationSourceDirectory.FullName, rules[0].SourceDirectory));
+		var sourceDirectory = Build.ReadFileSystem.DirectoryInfo.New(sourcePath);
+		return rules.Select(r =>
+		{
+			var file = Build.ReadFileSystem.FileInfo.New(Path.Combine(sourceDirectory.FullName, r.Path));
+			return defaultFileHandling(file, sourceDirectory);
+
+		}).ToArray();
+	}
+
+	public IReadOnlyCollection<ITocItem> CreateTableOfContentItems(
+		string parentPath,
+		string detectionRules,
+		HashSet<string> files
+	)
+	{
+		var detectionRulesFolder = $"{Path.Combine(parentPath, detectionRules)}".TrimStart('/');
+		var fs = Build.ReadFileSystem;
+		var sourceDirectory = Build.DocumentationSourceDirectory;
+		var path = fs.DirectoryInfo.New(fs.Path.GetFullPath(fs.Path.Combine(sourceDirectory.FullName, detectionRulesFolder)));
+		IReadOnlyCollection<ITocItem> children = path
+			.EnumerateFiles("*.*", SearchOption.AllDirectories)
+			.Where(f => !f.Attributes.HasFlag(FileAttributes.Hidden) && !f.Attributes.HasFlag(FileAttributes.System))
+			.Where(f => !f.Directory!.Attributes.HasFlag(FileAttributes.Hidden) && !f.Directory!.Attributes.HasFlag(FileAttributes.System))
+			.Where(f => f.Extension is ".md" or ".toml")
+			.Where(f => f.Name != "README.md")
+			.Select(f =>
+			{
+				var relativePath = Path.GetRelativePath(sourceDirectory.FullName, f.FullName);
+				if (f.Extension == ".toml")
+				{
+					var rule = DetectionRule.From(f);
+					return new RuleReference(relativePath, detectionRules, true, [], rule);
+				}
+
+				_ = files.Add(relativePath);
+				return new FileReference(relativePath, true, false, []);
+			})
+			.OrderBy(d => d is RuleReference r ? r.Rule.Name : null, StringComparer.OrdinalIgnoreCase)
+			.ToArray();
+
+		//return [new RulesFolderReference(detectionRulesFolder, true, children)];
+		return children;
+	}
+}

src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesReference.cs

@@ -0,0 +1,12 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using Elastic.Markdown.IO.Configuration;
+
+namespace Elastic.Markdown.Extensions.DetectionRules;
+
+public record RulesFolderReference(string Path, bool Found, IReadOnlyCollection<ITocItem> Children) : ITocItem;
+
+public record RuleReference(string Path, string SourceDirectory, bool Found, IReadOnlyCollection<ITocItem> Children, DetectionRule Rule)
+	: FileReference(Path, Found, false, Children);