Ensure links to directive anchors are validated

docs/source/docset.yml

@@ -1,8 +1,25 @@
 project: 'doc-builder'
+# docs-builder will warn for links to external hosts not declared here
+external_hosts:
+  - slack.com
+  - mystmd.org
+  - microsoft.com
+  - azure.com
+  - mistral.ai
+  - amazon.com
+  - python.org
+  - cohere.com
+  - docker.com
+  - langchain.com
+  - nodejs.org
+  - yarnpkg.com
+  - react.dev
+  - palletsprojects.com
 exclude:
   - '_*.md'
 toc:
   - file: index.md
+  - folder: markup
   - folder: elastic
     children:
       - file: index.md
@@ -24,7 +41,6 @@
             children:
               - file: search/req.md
               - file: search/setup.md
-  - folder: markup
   - folder: nested
     children:
       - folder: content

docs/source/elastic/search-labs/chat.md

@@ -4,8 +4,6 @@ title: Chatbot Tutorial
 
 In this tutorial you are going to build a large language model (LLM) chatbot that uses a pattern known as [Retrieval-Augmented Generation (RAG)](https://www.elastic.co/what-is/retrieval-augmented-generation).
 
-<!-- ![Chatbot App Demo](/assets/images/guides/chatbot-app-demo.gif) -->
-
 Chatbots built with RAG can overcome some of the limitations that general-purpose conversational models such as ChatGPT have. In particular, they are able to discuss and answer questions about:
 
 - Information that is private to your organization.

docs/source/elastic/search-labs/install/cloud.md

@@ -30,14 +30,10 @@ Follow these steps to obtain your Cloud ID:
 1. Locate your deployment, and click the **Manage** link under the **Actions** column.
 1. The Cloud ID is displayed on the right side of the page. See the screenshot below as a reference.
 
-<!-- ![My deployment](/assets/images/guides/elastic-cloud-my-deployment.png) -->
-
 ## Creating an API Key
 
 For security purposes, it is recommended that you create an API Key to use when authenticating to the Elasticsearch service.
 
-<!-- ![my deployment](/assets/images/guides/elastic-cloud-api-key.png) -->
-
 Follow these steps to create an API Key:
 
 1. Navigate to your [Elastic Cloud](https://cloud.elastic.co/home) home page.

docs/source/elastic/search-labs/search/setup.md

@@ -12,8 +12,6 @@ Download the starter search application by clicking on the link below.
 
 Find a suitable parent directory for your project, such as your *Documents* directory, and extract the contents of the zip file there. This should add a *search-tutorial* directory with several sub-directories and files inside.
 
-<!-- ![Search Starter App](/assets/images/guides/starter-app-structure.png) -->
-
 
 ## Install the Python Dependencies
 
@@ -69,8 +67,6 @@ flask run
 
 To confirm that the application is running, open your browser and navigate to http://localhost:5001.
 
-<!-- ![Search Starter App](/assets/images/guides/starter-app-screenshot.png) -->
-
 ```{note}
 The application in this early stage is just an empty shell. You can type something in the search box and request a search if you like, but the response is always going to be that there are no results. In the following sections you will learn how to load some content in an Elasticsearch index and perform searches.
 ```

docs/source/index.md

@@ -4,6 +4,8 @@ title: Elastic Docs v3
 
 Elastic Docs v3 is built with
 
+Test during zoom with colleen
+
 TODO ADD README for doc-builder
 
 :::{tip}

docs/source/markup/admonitions.md

@@ -6,39 +6,15 @@ Admonitions bring the attention of readers.
 
 ## Basic admonitions
 
-```{attention}
-:name: attention_ref
-This is an 'attention' admonition
-```
-
 ```{caution}
+:name: caution_ref
 This is a 'caution' admonition
 ```
 
-```{danger}
-This is a 'danger' admonition
-```
-
-```{error}
-This is an 'error' admonition
-```
-
-```{hint}
-This is an 'hint' admonition
-```
-
-```{important}
-This is an 'important' admonition
-```
-
 ```{note}
 This is a 'note' admonition
 ```
 
-```{seealso}
-This is a 'seealso' admonition
-```
-
 ```{tip}
 This is a tip
 ```
@@ -84,4 +60,4 @@ Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor i
 ## Link to admonitions
 You can add a 'name' option to an admonition, so that you can link to it elsewhere
 
-Here is a [link to attention](#attention_ref)
+Here is a [link to attention](#caution_ref)

src/Elastic.Markdown/BuildContext.cs

@@ -3,13 +3,20 @@
 // See the LICENSE file in the project root for more information
 using System.IO.Abstractions;
 using Elastic.Markdown.Diagnostics;
+using Elastic.Markdown.IO;
 
 namespace Elastic.Markdown;
 
 public record BuildContext
 {
-	public required IFileSystem ReadFileSystem { get; init; }
-	public required IFileSystem WriteFileSystem { get; init; }
+	public IFileSystem ReadFileSystem { get; }
+	public IFileSystem WriteFileSystem { get; }
+
+	public IDirectoryInfo SourcePath { get; }
+	public IDirectoryInfo OutputPath { get; }
+
+	public IFileInfo ConfigurationPath { get; }
+
 	public required DiagnosticsCollector Collector { get; init; }
 
 	public bool Force { get; init; }
@@ -22,4 +29,32 @@ public string? UrlPathPrefix
 
 	private readonly string? _urlPathPrefix;
 
+	public BuildContext(IFileSystem fileSystem)
+		: this(fileSystem, fileSystem, null, null) { }
+
+	public BuildContext(IFileSystem readFileSystem, IFileSystem writeFileSystem)
+		: this(readFileSystem, writeFileSystem, null, null) { }
+
+	public BuildContext(IFileSystem readFileSystem, IFileSystem writeFileSystem, string? source, string? output)
+	{
+		ReadFileSystem = readFileSystem;
+		WriteFileSystem = writeFileSystem;
+
+		SourcePath = !string.IsNullOrWhiteSpace(source)
+			? ReadFileSystem.DirectoryInfo.New(source)
+			: ReadFileSystem.DirectoryInfo.New(Path.Combine(Paths.Root.FullName, "docs/source"));
+		OutputPath = !string.IsNullOrWhiteSpace(output)
+			? WriteFileSystem.DirectoryInfo.New(output)
+			: WriteFileSystem.DirectoryInfo.New(Path.Combine(Paths.Root.FullName, ".artifacts/docs/html"));
+
+		ConfigurationPath =
+			SourcePath.EnumerateFiles("docset.yml", SearchOption.AllDirectories).FirstOrDefault()
+		    ?? ReadFileSystem.FileInfo.New(Path.Combine(SourcePath.FullName, "docset.yml"));
+
+		if (ConfigurationPath.FullName != SourcePath.FullName)
+			SourcePath = ConfigurationPath.Directory!;
+
+
+	}
+
 }

src/Elastic.Markdown/DocumentationGenerator.cs

@@ -33,36 +33,22 @@ public class DocumentationGenerator
 
 	public DocumentationGenerator(
 		DocumentationSet docSet,
-		BuildContext context,
 		ILoggerFactory logger
 	)
 	{
-		_readFileSystem = context.ReadFileSystem;
-		_writeFileSystem = context.WriteFileSystem;
+		_readFileSystem = docSet.Context.ReadFileSystem;
+		_writeFileSystem = docSet.Context.WriteFileSystem;
 		_logger = logger.CreateLogger(nameof(DocumentationGenerator));
 
 		DocumentationSet = docSet;
-		Context = context;
+		Context = docSet.Context;
 		HtmlWriter = new HtmlWriter(DocumentationSet, _writeFileSystem);
 
 		_logger.LogInformation($"Created documentation set for: {DocumentationSet.Name}");
 		_logger.LogInformation($"Source directory: {docSet.SourcePath} Exists: {docSet.SourcePath.Exists}");
 		_logger.LogInformation($"Output directory: {docSet.OutputPath} Exists: {docSet.OutputPath.Exists}");
 	}
 
-	public static DocumentationGenerator Create(
-		string? path,
-		string? output,
-		BuildContext context,
-		ILoggerFactory logger
-	)
-	{
-		var sourcePath = path != null ? context.ReadFileSystem.DirectoryInfo.New(path) : null;
-		var outputPath = output != null ? context.WriteFileSystem.DirectoryInfo.New(output) : null;
-		var docSet = new DocumentationSet(sourcePath, outputPath, context);
-		return new DocumentationGenerator(docSet, context, logger);
-	}
-
 	public OutputState? OutputState
 	{
 		get

src/Elastic.Markdown/IO/ConfigurationFile.cs

@@ -23,6 +23,11 @@ public record ConfigurationFile : DocumentationFile
 	public HashSet<string> Files { get; } = new(StringComparer.OrdinalIgnoreCase);
 	public HashSet<string> ImplicitFolders { get; } = new(StringComparer.OrdinalIgnoreCase);
 	public Glob[] Globs { get; } = [];
+	public HashSet<string> ExternalLinkHosts { get; } = new(StringComparer.OrdinalIgnoreCase)
+	{
+		"elastic.co",
+		"github.com",
+	};
 
 	public ConfigurationFile(IFileInfo sourceFile, IDirectoryInfo rootPath, BuildContext context)
 		: base(sourceFile, rootPath)
@@ -62,6 +67,12 @@ public ConfigurationFile(IFileInfo sourceFile, IDirectoryInfo rootPath, BuildCon
 						.Select(Glob.Parse)
 						.ToArray();
 					break;
+				case "external_hosts":
+					var hosts = ReadStringArray(entry)
+						.ToArray();
+					foreach (var host in hosts)
+						ExternalLinkHosts.Add(host);
+					break;
 				case "toc":
 					var entries = ReadChildren(entry, string.Empty);
 

src/Elastic.Markdown/IO/DocumentationSet.cs

@@ -10,41 +10,31 @@ namespace Elastic.Markdown.IO;
 
 public class DocumentationSet
 {
+	public BuildContext Context { get; }
 	public string Name { get; }
+	public IFileInfo OutputStateFile { get; }
+
 	public IDirectoryInfo SourcePath { get; }
 	public IDirectoryInfo OutputPath { get; }
-	public IFileInfo OutputStateFile { get; }
 
 	public DateTimeOffset LastWrite { get; }
 
 	public ConfigurationFile Configuration { get; }
 
 	public MarkdownParser MarkdownParser { get; }
 
-	public DocumentationSet(BuildContext context) : this(null, null, context) { }
-
-	public DocumentationSet(IDirectoryInfo? sourcePath, IDirectoryInfo? outputPath, BuildContext context)
+	public DocumentationSet(BuildContext context)
 	{
-		SourcePath = sourcePath ?? context.ReadFileSystem.DirectoryInfo.New(Path.Combine(Paths.Root.FullName, "docs/source"));
-		OutputPath = outputPath ?? context.WriteFileSystem.DirectoryInfo.New(Path.Combine(Paths.Root.FullName, ".artifacts/docs/html"));
+		Context = context;
+		SourcePath = context.SourcePath;
+		OutputPath = context.OutputPath;
+		Configuration = new ConfigurationFile(context.ConfigurationPath, SourcePath, context);
 
-		var configurationFile = SourcePath.EnumerateFiles("docset.yml", SearchOption.AllDirectories).FirstOrDefault();
-		if (configurationFile is null)
-		{
-			configurationFile = context.ReadFileSystem.FileInfo.New(Path.Combine(SourcePath.FullName, "docset.yml"));
-			context.EmitWarning(configurationFile, "No configuration file found");
-		}
-
-		if (configurationFile.Directory!.FullName != SourcePath.FullName)
-			SourcePath = configurationFile.Directory;
-
-		MarkdownParser = new MarkdownParser(SourcePath, context, GetTitle);
+		MarkdownParser = new MarkdownParser(SourcePath, context, GetMarkdownFile, Configuration);
 
 		Name = SourcePath.FullName;
 		OutputStateFile = OutputPath.FileSystem.FileInfo.New(Path.Combine(OutputPath.FullName, ".doc.state"));
 
-		Configuration = new ConfigurationFile(configurationFile, SourcePath, context);
-
 		Files = context.ReadFileSystem.Directory
 			.EnumerateFiles(SourcePath.FullName, "*.*", SearchOption.AllDirectories)
 			.Select(f => context.ReadFileSystem.FileInfo.New(f))
@@ -68,10 +58,9 @@ public DocumentationSet(IDirectoryInfo? sourcePath, IDirectoryInfo? outputPath,
 		Tree = new DocumentationFolder(Configuration.TableOfContents, FlatMappedFiles, folderFiles);
 	}
 
-	private string? GetTitle(string relativePath) => GetMarkdownFile(relativePath)?.YamlFrontMatter?.Title;
-
-	public MarkdownFile? GetMarkdownFile(string relativePath)
+	public MarkdownFile? GetMarkdownFile(IFileInfo sourceFile)
 	{
+		var relativePath = Path.GetRelativePath(SourcePath.FullName, sourceFile.FullName);
 		if (FlatMappedFiles.TryGetValue(relativePath, out var file) && file is MarkdownFile markdownFile)
 			return markdownFile;
 		return null;

src/Elastic.Markdown/IO/MarkdownFile.cs

@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information
 using System.IO.Abstractions;
 using Elastic.Markdown.Myst;
+using Elastic.Markdown.Myst.Directives;
 using Elastic.Markdown.Slices;
 using Markdig;
 using Markdig.Extensions.Yaml;
@@ -34,8 +35,12 @@ public string? NavigationTitle
 		private set => _navigationTitle = value;
 	}
 
-	private readonly List<PageTocItem> _tableOfContent = new();
-	public IReadOnlyCollection<PageTocItem> TableOfContents => _tableOfContent;
+	//indexed by slug
+	private readonly Dictionary<string, PageTocItem> _tableOfContent = new();
+	public IReadOnlyDictionary<string, PageTocItem> TableOfContents => _tableOfContent;
+
+	private readonly HashSet<string> _additionalLabels = new();
+	public IReadOnlySet<string> AdditionalLabels => _additionalLabels;
 
 	public string FileName { get; }
 	public string Url => $"{UrlPathPrefix}/{RelativePath.Replace(".md", ".html")}";
@@ -76,7 +81,20 @@ private void ReadDocumentInstructions(MarkdownDocument document)
 			.Select(title => new PageTocItem { Heading = title!, Slug = _slugHelper.GenerateSlug(title) })
 			.ToList();
 		_tableOfContent.Clear();
-		_tableOfContent.AddRange(contents);
+		foreach (var t in contents)
+			_tableOfContent[t.Slug] = t;
+
+		var labels = document.Descendants<DirectiveBlock>()
+			.Select(b=>b.CrossReferenceName)
+			.Where(l=>!string.IsNullOrWhiteSpace(l))
+			.Select(_slugHelper.GenerateSlug)
+			.ToArray();
+		foreach(var label in labels)
+		{
+			if (!string.IsNullOrEmpty(label))
+				_additionalLabels.Add(label);
+		}
+
 		_instructionsParsed = true;
 	}
 

src/Elastic.Markdown/Myst/Directives/AdmonitionBlock.cs

@@ -10,9 +10,7 @@ public class AdmonitionBlock(DirectiveBlockParser parser, string admonition, Dic
 
 	public override string Directive => Admonition;
 
-	public string? Label { get; protected set; }
 	public string? Classes { get; protected set; }
-	public string? CrossReferenceName  { get; private set; }
 	public bool? DropdownOpen  { get; private set; }
 
 	public string Title
@@ -31,7 +29,6 @@ public string Title
 
 	public override void FinalizeAndValidate(ParserContext context)
 	{
-		Label = Prop("label");
 		Classes = Properties.GetValueOrDefault("class");
 		CrossReferenceName = Properties.GetValueOrDefault("name");
 		DropdownOpen = PropBool("open");

src/Elastic.Markdown/Myst/Directives/CodeBlock.cs

@@ -8,7 +8,6 @@ public class CodeBlock(DirectiveBlockParser parser, string directive, Dictionary
 {
 	public override string Directive => directive;
 	public string? Caption { get; private set; }
-	public string? CrossReferenceName { get; private set; }
 
 	public string Language
 	{
@@ -23,6 +22,6 @@ public string Language
 	public override void FinalizeAndValidate(ParserContext context)
 	{
 		Caption = Properties.GetValueOrDefault("caption");
-		CrossReferenceName = Properties.GetValueOrDefault("name");
+		CrossReferenceName = Prop("name", "label");
 	}
 }