Refactor TOC handling and support nested TOC configurations (#749)

Introduced `TableOfContentsConfiguration` to refactor how TOC files are parsed and improve code modularity. Added support for a `max_toc_depth` parameter to restrict TOC nesting depth. Updated configurations and comments to clarify support for nested TOCs and modified file paths accordingly.

Author: Mpdreamz

docs/_docset.yml

@@ -1,4 +1,5 @@
 project: 'doc-builder'
+max_toc_depth: 2
 cross_links:
   - docs-content
 exclude:
@@ -92,8 +93,9 @@ toc:
     children:
       - file: index.md
       - file: content-patterns.md
-  # nested TOCs are only allowed from docset.yml
+  # nested TOCs are only allowed from docset.yml by default
   # to prevent them from being nested deeply arbitrarily
+  # use max_toc_depth to allow deeper nesting (Expert mode, consult with docs team)
   - toc: development
   - folder: testing
     children:

docs/development/link-validation.md renamed to docs/development/link-validation/link-validation.md

@@ -0,0 +1,2 @@
+toc:
+  - file: link-validation.md

docs/development/link-validation/toc.yml

@@ -1,3 +1,3 @@
 toc:
   - file: index.md
-  - file: link-validation.md
+  - toc: link-validation

docs/development/toc.yml

@@ -77,7 +77,7 @@ public BuildContext(DiagnosticsCollector collector, IFileSystem readFileSystem,
 			DocumentationSourceDirectory = ConfigurationPath.Directory!;
 
 		Git = GitCheckoutInformation.Create(DocumentationSourceDirectory, ReadFileSystem);
-		Configuration = new ConfigurationFile(ConfigurationPath, DocumentationSourceDirectory, this);
+		Configuration = new ConfigurationFile(this);
 
 	}
 

src/Elastic.Markdown/BuildContext.cs

@@ -2,37 +2,39 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
-using System.IO.Abstractions;
-using System.Runtime.InteropServices;
 using DotNet.Globbing;
 using Elastic.Markdown.Diagnostics;
 using Elastic.Markdown.Extensions;
 using Elastic.Markdown.Extensions.DetectionRules;
 using Elastic.Markdown.IO.State;
-using YamlDotNet.RepresentationModel;
 
 namespace Elastic.Markdown.IO.Configuration;
 
 public record ConfigurationFile : DocumentationFile
 {
-	private readonly IDirectoryInfo _rootPath;
 	private readonly BuildContext _context;
-	private readonly int _depth;
+
 	public string? Project { get; }
+
 	public Glob[] Exclude { get; } = [];
-	public bool SoftLineEndings { get; }
 
 	public string[] CrossLinkRepositories { get; } = [];
 
+	/// The maximum depth `toc.yml` files may appear
+	public int MaxTocDepth { get; } = 1;
+
 	public EnabledExtensions Extensions { get; } = new([]);
+
 	public IReadOnlyCollection<IDocsBuilderExtension> EnabledExtensions { get; } = [];
 
 	public IReadOnlyCollection<ITocItem> TableOfContents { get; } = [];
 
+	public HashSet<string> Files { get; } = new(StringComparer.OrdinalIgnoreCase);
+
 	public Dictionary<string, LinkRedirect>? Redirects { get; }
 
-	public HashSet<string> Files { get; } = new(StringComparer.OrdinalIgnoreCase);
 	public HashSet<string> ImplicitFolders { get; } = new(StringComparer.OrdinalIgnoreCase);
+
 	public Glob[] Globs { get; } = [];
 
 	private readonly Dictionary<string, string> _substitutions = new(StringComparer.OrdinalIgnoreCase);
@@ -42,19 +44,18 @@ public record ConfigurationFile : DocumentationFile
 	private FeatureFlags? _featureFlags;
 	public FeatureFlags Features => _featureFlags ??= new FeatureFlags(_features);
 
-	public ConfigurationFile(IFileInfo sourceFile, IDirectoryInfo rootPath, BuildContext context, int depth = 0, string parentPath = "")
-		: base(sourceFile, rootPath)
+	public ConfigurationFile(BuildContext context)
+		: base(context.ConfigurationPath, context.DocumentationSourceDirectory)
 	{
-		_rootPath = rootPath;
 		_context = context;
-		_depth = depth;
-		if (!sourceFile.Exists)
+		if (!context.ConfigurationPath.Exists)
 		{
 			Project = "unknown";
-			context.EmitWarning(sourceFile, "No configuration file found");
+			context.EmitWarning(context.ConfigurationPath, "No configuration file found");
 			return;
 		}
 
+		var sourceFile = context.ConfigurationPath;
 		var redirectFileName = sourceFile.Name.StartsWith('_') ? "_redirects.yml" : "redirects.yml";
 		var redirectFileInfo = sourceFile.FileSystem.FileInfo.New(Path.Combine(sourceFile.Directory!.FullName, redirectFileName));
 		var redirectFile = new RedirectFile(redirectFileInfo, _context);
@@ -70,11 +71,12 @@ public ConfigurationFile(IFileInfo sourceFile, IDirectoryInfo rootPath, BuildCon
 					case "project":
 						Project = reader.ReadString(entry.Entry);
 						break;
-					case "soft_line_endings":
-						SoftLineEndings = bool.TryParse(reader.ReadString(entry.Entry), out var softLineEndings) && softLineEndings;
+					case "max_toc_depth":
+						MaxTocDepth = int.TryParse(reader.ReadString(entry.Entry), out var maxTocDepth) ? maxTocDepth : 1;
 						break;
 					case "exclude":
-						Exclude = [.. YamlStreamReader.ReadStringArray(entry.Entry).Select(Glob.Parse)];
+						var excludes = YamlStreamReader.ReadStringArray(entry.Entry);
+						Exclude = [.. excludes.Where(s => !string.IsNullOrEmpty(s)).Select(Glob.Parse)];
 						break;
 					case "cross_links":
 						CrossLinkRepositories = [.. YamlStreamReader.ReadStringArray(entry.Entry)];
@@ -87,15 +89,7 @@ public ConfigurationFile(IFileInfo sourceFile, IDirectoryInfo rootPath, BuildCon
 						_substitutions = reader.ReadDictionary(entry.Entry);
 						break;
 					case "toc":
-						if (depth > 1)
-						{
-							reader.EmitError($"toc.yml files may only be linked from docset.yml", entry.Key);
-							break;
-						}
-
-						var entries = ReadChildren(reader, entry.Entry, parentPath);
-
-						TableOfContents = entries;
+						// read this later
 						break;
 					case "features":
 						_features = reader.ReadDictionary(entry.Entry).ToDictionary(k => k.Key, v => bool.Parse(v.Value), StringComparer.OrdinalIgnoreCase);
@@ -108,6 +102,21 @@ public ConfigurationFile(IFileInfo sourceFile, IDirectoryInfo rootPath, BuildCon
 						break;
 				}
 			}
+
+			//we read it twice to ensure we read 'toc' last
+			reader = new YamlStreamReader(sourceFile, _context);
+			foreach (var entry in reader.Read())
+			{
+				switch (entry.Key)
+				{
+					case "toc":
+						var toc = new TableOfContentsConfiguration(this, _context, 0, "");
+						var entries = toc.ReadChildren(reader, entry.Entry);
+						TableOfContents = entries;
+						Files = toc.Files; //side-effect ripe for refactor
+						break;
+				}
+			}
 		}
 		catch (Exception e)
 		{
@@ -135,195 +144,4 @@ private IReadOnlyCollection<IDocsBuilderExtension> InstantiateExtensions()
 	}
 
 
-	private List<ITocItem> ReadChildren(YamlStreamReader reader, KeyValuePair<YamlNode, YamlNode> entry, string parentPath)
-	{
-		var entries = new List<ITocItem>();
-		if (entry.Value is not YamlSequenceNode sequence)
-		{
-			if (entry.Key is YamlScalarNode scalarKey)
-			{
-				var key = scalarKey.Value;
-				reader.EmitWarning($"'{key}' is not an array");
-			}
-			else
-				reader.EmitWarning($"'{entry.Key}' is not an array");
-
-			return entries;
-		}
-
-		entries.AddRange(
-			sequence.Children.OfType<YamlMappingNode>()
-				.SelectMany(tocEntry => ReadChild(reader, tocEntry, parentPath) ?? [])
-		);
-
-		return entries;
-	}
-
-	private IEnumerable<ITocItem>? ReadChild(YamlStreamReader reader, YamlMappingNode tocEntry, string parentPath)
-	{
-		string? file = null;
-		string? folder = null;
-		string? detectionRules = null;
-		ConfigurationFile? toc = null;
-		var fileFound = false;
-		var folderFound = false;
-		var detectionRulesFound = false;
-		var hiddenFile = false;
-		var inNav = false;
-		IReadOnlyCollection<ITocItem>? children = null;
-		foreach (var entry in tocEntry.Children)
-		{
-			var key = ((YamlScalarNode)entry.Key).Value;
-			switch (key)
-			{
-				case "toc":
-					toc = ReadNestedToc(reader, entry, out fileFound);
-					break;
-				case "in_nav":
-					if (!bool.TryParse(reader.ReadString(entry), out inNav))
-						throw new ArgumentException("in_nav must be a boolean");
-					break;
-				case "hidden":
-				case "file":
-					hiddenFile = key == "hidden";
-					file = ReadFile(reader, entry, parentPath, out fileFound);
-					break;
-				case "folder":
-					folder = ReadFolder(reader, entry, parentPath, out folderFound);
-					parentPath += $"{Path.DirectorySeparatorChar}{folder}";
-					break;
-				case "detection_rules":
-					if (Extensions.IsDetectionRulesEnabled)
-					{
-						detectionRules = ReadDetectionRules(reader, entry, parentPath, out detectionRulesFound);
-						parentPath += $"{Path.DirectorySeparatorChar}{folder}";
-					}
-					break;
-				case "children":
-					children = ReadChildren(reader, entry, parentPath);
-					break;
-			}
-		}
-
-		if (toc is not null)
-		{
-			foreach (var f in toc.Files)
-				_ = Files.Add(f);
-
-			return [new FolderReference($"{parentPath}".TrimStart(Path.DirectorySeparatorChar), folderFound, inNav, toc.TableOfContents)];
-		}
-
-		if (file is not null)
-		{
-			if (detectionRules is not null)
-			{
-				if (children is not null)
-					reader.EmitError($"'detection_rules' is not allowed to have 'children'", tocEntry);
-
-				if (!detectionRulesFound)
-				{
-					reader.EmitError($"'detection_rules' folder {parentPath} is not found, skipping'", tocEntry);
-					children = [];
-				}
-				else
-				{
-					var extension = EnabledExtensions.OfType<DetectionRulesDocsBuilderExtension>().First();
-					children = extension.CreateTableOfContentItems(parentPath, detectionRules, Files);
-				}
-			}
-			return [new FileReference($"{parentPath}{Path.DirectorySeparatorChar}{file}".TrimStart(Path.DirectorySeparatorChar), fileFound, hiddenFile, children ?? [])];
-		}
-
-		if (folder is not null)
-		{
-			if (children is null)
-				_ = ImplicitFolders.Add(parentPath.TrimStart(Path.DirectorySeparatorChar));
-
-			return [new FolderReference($"{parentPath}".TrimStart(Path.DirectorySeparatorChar), folderFound, inNav, children ?? [])];
-		}
-
-		return null;
-	}
-
-	private string? ReadFolder(YamlStreamReader reader, KeyValuePair<YamlNode, YamlNode> entry, string parentPath, out bool found)
-	{
-		found = false;
-		var folder = reader.ReadString(entry);
-		if (folder is not null)
-		{
-			var path = Path.Combine(_rootPath.FullName, parentPath.TrimStart(Path.DirectorySeparatorChar), folder);
-			if (!_context.ReadFileSystem.DirectoryInfo.New(path).Exists)
-				reader.EmitError($"Directory '{path}' does not exist", entry.Key);
-			else
-				found = true;
-		}
-
-		return folder;
-	}
-
-	private string? ReadDetectionRules(YamlStreamReader reader, KeyValuePair<YamlNode, YamlNode> entry, string parentPath, out bool found)
-	{
-		found = false;
-		var folder = reader.ReadString(entry);
-		if (folder is not null)
-		{
-			var path = Path.Combine(_rootPath.FullName, parentPath.TrimStart(Path.DirectorySeparatorChar), folder);
-			if (!_context.ReadFileSystem.DirectoryInfo.New(path).Exists)
-				reader.EmitError($"Directory '{path}' does not exist", entry.Key);
-			else
-				found = true;
-		}
-
-		return folder;
-	}
-
-	private string? ReadFile(YamlStreamReader reader, KeyValuePair<YamlNode, YamlNode> entry, string parentPath, out bool found)
-	{
-		found = false;
-		var file = reader.ReadString(entry);
-		if (file is null)
-			return null;
-		if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
-			file = file.Replace('/', Path.DirectorySeparatorChar);
-
-		var path = Path.Combine(_rootPath.FullName, parentPath.TrimStart(Path.DirectorySeparatorChar), file);
-		if (!_context.ReadFileSystem.FileInfo.New(path).Exists)
-			reader.EmitError($"File '{path}' does not exist", entry.Key);
-		else
-			found = true;
-		_ = Files.Add((parentPath + Path.DirectorySeparatorChar + file).TrimStart(Path.DirectorySeparatorChar));
-
-		return file;
-	}
-
-	private ConfigurationFile? ReadNestedToc(YamlStreamReader reader, KeyValuePair<YamlNode, YamlNode> entry, out bool found)
-	{
-		found = false;
-		var tocPath = reader.ReadString(entry);
-		if (tocPath is null)
-		{
-			reader.EmitError($"Empty toc: reference", entry.Key);
-			return null;
-		}
-
-		var rootPath = _context.ReadFileSystem.DirectoryInfo.New(Path.Combine(_rootPath.FullName, tocPath));
-		var path = Path.Combine(rootPath.FullName, "toc.yml");
-		var source = _context.ReadFileSystem.FileInfo.New(path);
-
-		var errorMessage = $"Nested toc: '{source.Directory}' directory has no toc.yml or _toc.yml file";
-
-		if (!source.Exists)
-		{
-			path = Path.Combine(rootPath.FullName, "_toc.yml");
-			source = _context.ReadFileSystem.FileInfo.New(path);
-		}
-
-		if (!source.Exists)
-			reader.EmitError(errorMessage, entry.Key);
-		else
-			found = true;
-
-		var nestedConfiguration = new ConfigurationFile(source, _rootPath, _context, _depth + 1, tocPath);
-		return nestedConfiguration;
-	}
 }