feature/assembler global toc

docs-builder.sln

@@ -73,6 +73,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "update-reference-index", "u
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "docs-lambda-index-publisher", "src\docs-lambda-index-publisher\docs-lambda-index-publisher.csproj", "{C559D52D-100B-4B2B-BE87-2344D835761D}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "docs-assembler.Tests", "tests\docs-assembler.Tests\src\docs-assembler.Tests\docs-assembler.Tests.csproj", "{CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -122,6 +124,10 @@ Global
 		{C559D52D-100B-4B2B-BE87-2344D835761D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C559D52D-100B-4B2B-BE87-2344D835761D}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C559D52D-100B-4B2B-BE87-2344D835761D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{CDC0ECF4-6597-4FBA-8D25-5C244F0877E3}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{4D198E25-C211-41DC-9E84-B15E89BD7048} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
@@ -138,5 +144,6 @@ Global
 		{6554F917-73CE-4B3D-9101-F28EAA762C6B} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
 		{9FEC15F6-13F8-40B1-A66A-EB054E49E680} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
 		{C559D52D-100B-4B2B-BE87-2344D835761D} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
+		{CDC0ECF4-6597-4FBA-8D25-5C244F0877E3} = {67B576EE-02FA-4F9B-94BC-3630BC09ECE5}
 	EndGlobalSection
 EndGlobal

docs/_docset.yml

@@ -1,5 +1,8 @@
 project: 'doc-builder'
 max_toc_depth: 2
+# indicates this documentation set is not linkable by assembler.
+# relaxes a few restrictions around toc building and file placement
+dev_docs: true
 cross_links:
   - docs-content
 exclude:

docs/development/index.md

@@ -4,4 +4,7 @@
 
 # Development Guide
 
-TODO write development documentation here
+TODO write development documentation here
+
+
+![Image outside of scope](../images/great-drawing-of-new-structure.png)

src/Elastic.Documentation.Tooling/Diagnostics/Console/ConsoleDiagnosticsCollector.cs

@@ -24,7 +24,7 @@ protected override void HandleItem(Diagnostic diagnostic)
 			_errors.Add(diagnostic);
 		else if (diagnostic.Severity == Severity.Warning)
 			_warnings.Add(diagnostic);
-		else
+		else if (!NoHints)
 			_hints.Add(diagnostic);
 	}
 

src/Elastic.Markdown/Assets/markdown/table.css

@@ -1,8 +1,21 @@
+
 @layer components {
+
 	.table-wrapper {
 
-		@apply max-w-full overflow-x-scroll my-4;
+		@apply w-full overflow-x-scroll my-4;
+
+		&::-webkit-scrollbar {
+			@apply h-2 bg-grey-10 border-b-1 border-grey-20;
+		}
+		&::-webkit-scrollbar-thumb {
+			@apply bg-grey-80 rounded-full;
+		}
 
+		&::-webkit-scrollbar-thumb:hover {
+			@apply bg-grey-100;
+		}
+
 		table {
 			@apply w-full border-collapse border-1 border-grey-20;
 		}
@@ -13,6 +26,9 @@
 
 		thead {
 			@apply font-sans font-semibold text-left align-top border-b-1 border-grey-20 bg-grey-10;
+			:empty {
+				display: none;
+			}
 		}
 
 		tbody {

src/Elastic.Markdown/BuildContext.cs

@@ -29,6 +29,8 @@ public record BuildContext
 
 	public bool Force { get; init; }
 
+	public bool SkipMetadata { get; init; }
+
 	// This property is used to determine if the site should be indexed by search engines
 	public bool AllowIndexing { get; init; }
 
@@ -39,15 +41,6 @@ public string? UrlPathPrefix
 		init => _urlPathPrefix = value;
 	}
 
-	private readonly string? _staticUrlPathPrefix;
-	public string? StaticUrlPathPrefix
-	{
-		get => !string.IsNullOrWhiteSpace(_staticUrlPathPrefix)
-			? $"/{_staticUrlPathPrefix.Trim('/')}"
-			: UrlPathPrefix;
-		init => _staticUrlPathPrefix = value;
-	}
-
 	public BuildContext(IFileSystem fileSystem)
 		: this(new DiagnosticsCollector([]), fileSystem, fileSystem, null, null) { }
 

src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs

@@ -45,7 +45,7 @@ public interface ICrossLinkResolver
 public class CrossLinkResolver(CrossLinkFetcher fetcher, IUriEnvironmentResolver? uriResolver = null) : ICrossLinkResolver
 {
 	private FetchedCrossLinks _crossLinks = FetchedCrossLinks.Empty;
-	private readonly IUriEnvironmentResolver _uriResolver = uriResolver ?? new PreviewEnvironmentUriResolver();
+	private readonly IUriEnvironmentResolver _uriResolver = uriResolver ?? new IsolatedBuildEnvironmentUriResolver();
 
 	public async Task<FetchedCrossLinks> FetchLinks()
 	{

src/Elastic.Markdown/CrossLinks/IUriEnvironmentResolver.cs

@@ -9,7 +9,7 @@ public interface IUriEnvironmentResolver
 	Uri Resolve(Uri crossLinkUri, string path);
 }
 
-public class PreviewEnvironmentUriResolver : IUriEnvironmentResolver
+public class IsolatedBuildEnvironmentUriResolver : IUriEnvironmentResolver
 {
 	private static Uri BaseUri { get; } = new("https://docs-v3-preview.elastic.dev");
 

src/Elastic.Markdown/Diagnostics/DiagnosticsChannel.cs

@@ -85,10 +85,12 @@ public class DiagnosticsCollector(IReadOnlyCollection<IDiagnosticsOutput> output
 
 	public HashSet<string> OffendingFiles { get; } = [];
 
-	public HashSet<string> InUseSubstitutionKeys { get; } = [];
+	public ConcurrentDictionary<string, bool> InUseSubstitutionKeys { get; } = [];
 
 	public ConcurrentBag<string> CrossLinks { get; } = [];
 
+	public bool NoHints { get; init; }
+
 	public Task StartAsync(Cancel cancellationToken)
 	{
 		if (_started is not null)
@@ -117,6 +119,8 @@ void Drain()
 		{
 			while (Channel.Reader.TryRead(out var item))
 			{
+				if (item.Severity == Severity.Hint && NoHints)
+					continue;
 				IncrementSeverityCount(item);
 				HandleItem(item);
 				_ = OffendingFiles.Add(item.File);
@@ -132,7 +136,7 @@ private void IncrementSeverityCount(Diagnostic item)
 			_ = Interlocked.Increment(ref _errors);
 		else if (item.Severity == Severity.Warning)
 			_ = Interlocked.Increment(ref _warnings);
-		else if (item.Severity == Severity.Hint)
+		else if (item.Severity == Severity.Hint && !NoHints)
 			_ = Interlocked.Increment(ref _hints);
 	}
 
@@ -175,5 +179,5 @@ public async ValueTask DisposeAsync()
 	}
 
 	public void CollectUsedSubstitutionKey(ReadOnlySpan<char> key) =>
-		_ = InUseSubstitutionKeys.Add(key.ToString());
+		_ = InUseSubstitutionKeys.TryAdd(key.ToString(), true);
 }

src/Elastic.Markdown/Diagnostics/ProcessorDiagnosticExtensions.cs

@@ -98,6 +98,28 @@ public static void EmitWarning(this BuildContext context, IFileInfo file, string
 		context.Collector.Channel.Write(d);
 	}
 
+	public static void EmitError(this DiagnosticsCollector collector, IFileInfo file, string message, Exception? e = null)
+	{
+		var d = new Diagnostic
+		{
+			Severity = Severity.Error,
+			File = file.FullName,
+			Message = message + (e != null ? Environment.NewLine + e : string.Empty),
+		};
+		collector.Channel.Write(d);
+	}
+
+	public static void EmitWarning(this DiagnosticsCollector collector, IFileInfo file, string message)
+	{
+		var d = new Diagnostic
+		{
+			Severity = Severity.Warning,
+			File = file.FullName,
+			Message = message,
+		};
+		collector.Channel.Write(d);
+	}
+
 	public static void EmitError(this IBlockExtension block, string message, Exception? e = null)
 	{
 		if (block.SkipValidation)
@@ -132,8 +154,7 @@ public static void EmitWarning(this IBlockExtension block, string message)
 		block.Build.Collector.Channel.Write(d);
 	}
 
-
-	public static void EmitError(this InlineProcessor processor, LinkInline inline, string message)
+	private static void LinkDiagnostic(InlineProcessor processor, Severity severity, LinkInline inline, string message)
 	{
 		var url = inline.Url;
 		var line = inline.Line + 1;
@@ -145,36 +166,22 @@ public static void EmitError(this InlineProcessor processor, LinkInline inline,
 			return;
 		var d = new Diagnostic
 		{
-			Severity = Severity.Error,
+			Severity = severity,
 			File = processor.GetContext().MarkdownSourcePath.FullName,
-			Column = column,
+			Column = Math.Max(column, 1),
 			Line = line,
 			Message = message,
 			Length = length
 		};
 		context.Build.Collector.Channel.Write(d);
 	}
 
+	public static void EmitError(this InlineProcessor processor, LinkInline inline, string message) =>
+		LinkDiagnostic(processor, Severity.Error, inline, message);
 
-	public static void EmitWarning(this InlineProcessor processor, LinkInline inline, string message)
-	{
-		var url = inline.Url;
-		var line = inline.Line + 1;
-		var column = inline.Column;
-		var length = url?.Length ?? 1;
+	public static void EmitWarning(this InlineProcessor processor, LinkInline inline, string message) =>
+		LinkDiagnostic(processor, Severity.Warning, inline, message);
 
-		var context = processor.GetContext();
-		if (context.SkipValidation)
-			return;
-		var d = new Diagnostic
-		{
-			Severity = Severity.Warning,
-			File = processor.GetContext().MarkdownSourcePath.FullName,
-			Column = column,
-			Line = line,
-			Message = message,
-			Length = length
-		};
-		context.Build.Collector.Channel.Write(d);
-	}
+	public static void EmitHint(this InlineProcessor processor, LinkInline inline, string message) =>
+		LinkDiagnostic(processor, Severity.Hint, inline, message);
 }

src/Elastic.Markdown/DocumentationGenerator.cs

@@ -20,8 +20,14 @@ public interface IConversionCollector
 	void Collect(MarkdownFile file, MarkdownDocument document, string html);
 }
 
+public interface IDocumentationFileOutputProvider
+{
+	IFileInfo? OutputFile(DocumentationSet documentationSet, IFileInfo defaultOutputFile, string relativePath);
+}
+
 public class DocumentationGenerator
 {
+	private readonly IDocumentationFileOutputProvider? _documentationFileOutputProvider;
 	private readonly ILogger _logger;
 	private readonly IFileSystem _writeFileSystem;
 	private readonly IDocumentationFileExporter _documentationFileExporter;
@@ -34,10 +40,12 @@ public class DocumentationGenerator
 	public DocumentationGenerator(
 		DocumentationSet docSet,
 		ILoggerFactory logger,
+		IDocumentationFileOutputProvider? documentationFileOutputProvider = null,
 		IDocumentationFileExporter? documentationExporter = null,
 		IConversionCollector? conversionCollector = null
 	)
 	{
+		_documentationFileOutputProvider = documentationFileOutputProvider;
 		_writeFileSystem = docSet.Build.WriteFileSystem;
 		_logger = logger.CreateLogger(nameof(DocumentationGenerator));
 
@@ -74,7 +82,7 @@ public async Task ResolveDirectoryTree(Cancel ctx)
 	public async Task GenerateAll(Cancel ctx)
 	{
 		var generationState = GetPreviousGenerationState();
-		if (Context.Force || generationState == null)
+		if (!Context.SkipMetadata && (Context.Force || generationState == null))
 			DocumentationSet.ClearOutputDirectory();
 
 		if (CompilationNotNeeded(generationState, out var offendingFiles, out var outputSeenChanges))
@@ -91,17 +99,22 @@ public async Task GenerateAll(Cancel ctx)
 
 		await ExtractEmbeddedStaticResources(ctx);
 
-		_logger.LogInformation($"Completing diagnostics channel");
-		Context.Collector.Channel.TryComplete();
+		if (Context.SkipMetadata)
+			return;
 
 		_logger.LogInformation($"Generating documentation compilation state");
 		await GenerateDocumentationState(ctx);
 
 		_logger.LogInformation($"Generating links.json");
 		await GenerateLinkReference(ctx);
+	}
 
+	public async Task StopDiagnosticCollection(Cancel ctx)
+	{
 		_logger.LogInformation($"Completing diagnostics channel");
+		Context.Collector.Channel.TryComplete();
 
+		_logger.LogInformation($"Stopping diagnostics collector");
 		await Context.Collector.StopAsync(ctx);
 
 		_logger.LogInformation($"Completed diagnostics channel");
@@ -140,7 +153,8 @@ await Parallel.ForEachAsync(DocumentationSet.Files, ctx, async (file, token) =>
 	private void HintUnusedSubstitutionKeys()
 	{
 		var definedKeys = new HashSet<string>(Context.Configuration.Substitutions.Keys.ToArray());
-		var keysNotInUse = definedKeys.Except(Context.Collector.InUseSubstitutionKeys).ToArray();
+		var inUse = new HashSet<string>(Context.Collector.InUseSubstitutionKeys.Keys);
+		var keysNotInUse = definedKeys.Except(inUse).ToArray();
 		// If we have less than 20 unused keys emit them separately
 		// Otherwise emit one hint with all of them for brevity
 		if (keysNotInUse.Length >= 20)
@@ -170,6 +184,8 @@ private async Task ExtractEmbeddedStaticResources(Cancel ctx)
 			var path = a.Replace("Elastic.Markdown.", "").Replace("_static.", $"_static{Path.DirectorySeparatorChar}");
 
 			var outputFile = OutputFile(path);
+			if (outputFile is null)
+				continue;
 			await _documentationFileExporter.CopyEmbeddedResource(outputFile, resourceStream, ctx);
 			_logger.LogDebug("Copied static embedded resource {Path}", path);
 		}
@@ -186,14 +202,21 @@ private async Task ProcessFile(HashSet<string> offendingFiles, DocumentationFile
 		}
 
 		_logger.LogTrace("--> {FileFullPath}", file.SourceFile.FullName);
+		//TODO send file to OutputFile() so we can validate its scope is defined in navigation.yml
 		var outputFile = OutputFile(file.RelativePath);
-		await _documentationFileExporter.ProcessFile(file, outputFile, token);
+		if (outputFile is not null)
+			await _documentationFileExporter.ProcessFile(file, outputFile, token);
 	}
 
-	private IFileInfo OutputFile(string relativePath)
+	private IFileInfo? OutputFile(string relativePath)
 	{
 		var outputFile = _writeFileSystem.FileInfo.New(Path.Combine(DocumentationSet.OutputDirectory.FullName, relativePath));
-		return outputFile;
+		if (relativePath.StartsWith("_static"))
+			return outputFile;
+
+		return _documentationFileOutputProvider is not null
+			? _documentationFileOutputProvider.OutputFile(DocumentationSet, outputFile, relativePath)
+			: outputFile;
 	}
 
 	private bool CompilationNotNeeded(GenerationState? generationState, out HashSet<string> offendingFiles,

src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesDocsBuilderExtension.cs

@@ -97,11 +97,11 @@ HashSet<string> files
 				if (f.Extension == ".toml")
 				{
 					var rule = DetectionRule.From(f);
-					return new RuleReference(relativePath, detectionRules, true, [], rule);
+					return new RuleReference(Build.Configuration, relativePath, detectionRules, true, [], rule);
 				}
 
 				_ = files.Add(relativePath);
-				return new FileReference(relativePath, true, false, []);
+				return new FileReference(Build.Configuration, relativePath, true, false, []);
 			})
 			.OrderBy(d => d is RuleReference r ? r.Rule.Name : null, StringComparer.OrdinalIgnoreCase)
 			.ToArray();

src/Elastic.Markdown/Extensions/DetectionRules/DetectionRulesReference.cs

@@ -6,7 +6,13 @@
 
 namespace Elastic.Markdown.Extensions.DetectionRules;
 
-public record RulesFolderReference(string Path, bool Found, IReadOnlyCollection<ITocItem> Children) : ITocItem;
+public record RulesFolderReference(ITableOfContentsScope TableOfContentsScope, string Path, bool Found, IReadOnlyCollection<ITocItem> Children) : ITocItem;
 
-public record RuleReference(string Path, string SourceDirectory, bool Found, IReadOnlyCollection<ITocItem> Children, DetectionRule Rule)
-	: FileReference(Path, Found, false, Children);
+public record RuleReference(
+	ITableOfContentsScope TableOfContentsScope,
+	string Path,
+	string SourceDirectory,
+	bool Found,
+	IReadOnlyCollection<ITocItem> Children, DetectionRule Rule
+)
+	: FileReference(TableOfContentsScope, Path, Found, false, Children);