Add inline link/image validation

src/Elastic.Markdown/Diagnostics/ProcessorDiagnosticExtensions.cs

@@ -28,7 +28,7 @@ public static void EmitError(this InlineProcessor processor, int line, int colum
 	}
 
 
-	public static void EmitWarning(this BlockProcessor processor, int line, int column, int length, string message)
+	public static void EmitWarning(this InlineProcessor processor, int line, int column, int length, string message)
 	{
 		var context = processor.GetContext();
 		if (context.SkipValidation) return;

src/Elastic.Markdown/Elastic.Markdown.csproj

@@ -26,8 +26,4 @@
     <PackageReference Include="System.IO.Abstractions" Version="21.0.29" />
   </ItemGroup>
 
-  <ItemGroup>
-    <Folder Include="Myst\Inline\" />
-  </ItemGroup>
-
 </Project>

src/Elastic.Markdown/IO/DocumentationSet.cs

@@ -19,19 +19,30 @@ public class DocumentationSet
 
 	public ConfigurationFile Configuration { get; }
 
-	private MarkdownParser MarkdownParser { get; }
+	public MarkdownParser MarkdownParser { get; }
 
 	public DocumentationSet(BuildContext context) : this(null, null, context) { }
 
 	public DocumentationSet(IDirectoryInfo? sourcePath, IDirectoryInfo? outputPath, BuildContext context)
 	{
 		SourcePath = sourcePath ?? context.ReadFileSystem.DirectoryInfo.New(Path.Combine(Paths.Root.FullName, "docs/source"));
 		OutputPath = outputPath ?? context.WriteFileSystem.DirectoryInfo.New(Path.Combine(Paths.Root.FullName, ".artifacts/docs/html"));
+
+		var configurationFile = SourcePath.EnumerateFiles("docset.yml", SearchOption.AllDirectories).FirstOrDefault();
+		if (configurationFile is null)
+		{
+			configurationFile = context.ReadFileSystem.FileInfo.New(Path.Combine(SourcePath.FullName, "docset.yml"));
+			context.EmitWarning(configurationFile, "No configuration file found");
+		}
+
+		if (configurationFile.Directory!.FullName != SourcePath.FullName)
+			SourcePath = configurationFile.Directory;
+
+		MarkdownParser = new MarkdownParser(SourcePath, context, GetTitle);
+
 		Name = SourcePath.FullName;
-		MarkdownParser = new MarkdownParser(SourcePath, context);
 		OutputStateFile = OutputPath.FileSystem.FileInfo.New(Path.Combine(OutputPath.FullName, ".doc.state"));
 
-		var configurationFile = context.ReadFileSystem.FileInfo.New(Path.Combine(SourcePath.FullName, "docset.yml"));
 		Configuration = new ConfigurationFile(configurationFile, SourcePath, context);
 
 		Files = context.ReadFileSystem.Directory
@@ -57,6 +68,18 @@ public DocumentationSet(IDirectoryInfo? sourcePath, IDirectoryInfo? outputPath,
 		Tree = new DocumentationFolder(Configuration.TableOfContents, FlatMappedFiles, folderFiles);
 	}
 
+	private string? GetTitle(string relativePath) => GetMarkdownFile(relativePath)?.YamlFrontMatter?.Title;
+
+	public MarkdownFile? GetMarkdownFile(string relativePath)
+	{
+		if (FlatMappedFiles.TryGetValue(relativePath, out var file) && file is MarkdownFile markdownFile)
+			return markdownFile;
+		return null;
+	}
+
+	public async Task ResolveDirectoryTree(Cancel ctx) =>
+		await Tree.Resolve(ctx);
+
 	private DocumentationFile CreateMarkDownFile(IFileInfo file, BuildContext context)
 	{
 		if (Configuration.Exclude.Any(g => g.IsMatch(file.Name)))

src/Elastic.Markdown/Myst/Directives/DirectiveHtmlRenderer.cs

@@ -208,7 +208,7 @@ private void WriteIncludeBlock(HtmlRenderer renderer, IncludeBlock block)
 		if (!block.Found || block.IncludePath is null)
 			return;
 
-		var parser = new MarkdownParser(block.DocumentationSourcePath, block.Build);
+		var parser = new MarkdownParser(block.DocumentationSourcePath, block.Build, block.GetTitle);
 		var file = block.FileSystem.FileInfo.New(block.IncludePath);
 		var document = parser.ParseAsync(file, block.FrontMatter, default).GetAwaiter().GetResult();
 		var html = document.ToHtml(parser.Pipeline);

src/Elastic.Markdown/Myst/Directives/IncludeBlock.cs

@@ -13,6 +13,8 @@ public class IncludeBlock(DirectiveBlockParser parser, Dictionary<string, string
 
 	public BuildContext Build { get; } = context.Build;
 
+	public Func<string, string?>? GetTitle { get; } = context.GetTitle;
+
 	public IFileSystem FileSystem { get; } = context.Build.ReadFileSystem;
 
 	public IDirectoryInfo DocumentationSourcePath { get; } = context.Parser.SourcePath;

src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs

@@ -0,0 +1,87 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using Elastic.Markdown.Diagnostics;
+using Elastic.Markdown.Myst.Directives;
+using Markdig;
+using Markdig.Helpers;
+using Markdig.Parsers;
+using Markdig.Parsers.Inlines;
+using Markdig.Renderers;
+using Markdig.Syntax.Inlines;
+
+namespace Elastic.Markdown.Myst.InlineParsers;
+
+public static class DirectiveMarkdownBuilderExtensions
+{
+	public static MarkdownPipelineBuilder UseDiagnosticLinks(this MarkdownPipelineBuilder pipeline)
+	{
+		pipeline.Extensions.AddIfNotAlready<DiagnosticLinkInlineExtensions>();
+		return pipeline;
+	}
+}
+
+public class DiagnosticLinkInlineExtensions : IMarkdownExtension
+{
+	public void Setup(MarkdownPipelineBuilder pipeline) =>
+		pipeline.InlineParsers.Replace<LinkInlineParser>(new DiagnosticLinkInlineParser());
+
+	public void Setup(MarkdownPipeline pipeline, IMarkdownRenderer renderer) { }
+}
+
+public class DiagnosticLinkInlineParser : LinkInlineParser
+{
+	public override bool Match(InlineProcessor processor, ref StringSlice slice)
+	{
+		var match = base.Match(processor, ref slice);
+		if (!match) return false;
+
+		if (processor.Inline is not LinkInline link)
+			return match;
+
+		var url = link.Url;
+		var line = link.Line + 1;
+		var column = link.Column;
+		var length = url?.Length ?? 1;
+
+		var context = processor.GetContext();
+		if (processor.GetContext().SkipValidation)
+			return match;
+
+		if (string.IsNullOrEmpty(url))
+		{
+			processor.EmitWarning(line, column, length, $"Found empty url");
+			return match;
+		}
+
+		if (Uri.TryCreate(url, UriKind.Absolute, out var uri) && uri.Scheme.StartsWith("http"))
+		{
+			processor.EmitWarning(line, column, length, $"external URI: {uri} ");
+			return match;
+		}
+
+		var includeFrom = context.Path.Directory!.FullName;
+		if (url.StartsWith('/'))
+			includeFrom = context.Parser.SourcePath.FullName;
+
+		var pathOnDisk = Path.Combine(includeFrom, url.TrimStart('/'));
+		if (!context.Build.ReadFileSystem.File.Exists(pathOnDisk))
+			processor.EmitError(line, column, length, $"`{url}` does not exist. resolved to `{pathOnDisk}");
+
+		if (link.FirstChild == null)
+		{
+			var title = context.GetTitle?.Invoke(url);
+			if (!string.IsNullOrEmpty(title))
+				link.AppendChild(new LiteralInline(title));
+		}
+
+		if (url.EndsWith(".md"))
+			link.Url = Path.ChangeExtension(url, ".html");
+
+		return match;
+
+
+
+	}
+}

src/Elastic.Markdown/Myst/MarkdownParser.cs

@@ -6,28 +6,31 @@
 using Cysharp.IO;
 using Elastic.Markdown.Myst.Comments;
 using Elastic.Markdown.Myst.Directives;
+using Elastic.Markdown.Myst.InlineParsers;
 using Elastic.Markdown.Myst.Substitution;
 using Markdig;
 using Markdig.Extensions.EmphasisExtras;
 using Markdig.Syntax;
 
 namespace Elastic.Markdown.Myst;
 
-public class MarkdownParser(IDirectoryInfo sourcePath, BuildContext context)
+public class MarkdownParser(IDirectoryInfo sourcePath, BuildContext context, Func<string, string?>? getTitle)
 {
 	public IDirectoryInfo SourcePath { get; } = sourcePath;
 	public BuildContext Context { get; } = context;
 
 	public MarkdownPipeline MinimalPipeline { get; } =
 		new MarkdownPipelineBuilder()
+			.UseDiagnosticLinks()
 			.UseSubstitution()
 			.UseYamlFrontMatter()
 			.Build();
 
-	public MarkdownPipeline Pipeline { get; } =
+	public MarkdownPipeline Pipeline =>
 		new MarkdownPipelineBuilder()
 			.EnableTrackTrivia()
 			.UsePreciseSourceLocation()
+			.UseDiagnosticLinks()
 			.UseGenericAttributes()
 			.UseEmphasisExtras(EmphasisExtraOptions.Default)
 			.UseSoftlineBreakAsHardlineBreak()
@@ -43,13 +46,20 @@ public class MarkdownParser(IDirectoryInfo sourcePath, BuildContext context)
 
 	public Task<MarkdownDocument> MinimalParseAsync(IFileInfo path, Cancel ctx)
 	{
-		var context = new ParserContext(this, path, null, Context) { SkipValidation = true };
+		var context = new ParserContext(this, path, null, Context)
+		{
+			SkipValidation = true,
+			GetTitle = getTitle
+		};
 		return ParseAsync(path, context, MinimalPipeline, ctx);
 	}
 
 	public Task<MarkdownDocument> ParseAsync(IFileInfo path, YamlFrontMatter? matter, Cancel ctx)
 	{
-		var context = new ParserContext(this, path, matter, Context);
+		var context = new ParserContext(this, path, matter, Context)
+		{
+			GetTitle = getTitle
+		};
 		return ParseAsync(path, context, Pipeline, ctx);
 	}
 

src/Elastic.Markdown/Myst/ParserContext.cs

@@ -52,4 +52,5 @@ public ParserContext(MarkdownParser markdownParser,
 	public YamlFrontMatter? FrontMatter { get; }
 	public BuildContext Build { get; }
 	public bool SkipValidation { get; init; }
+	public Func<string, string?>? GetTitle { get; init; }
 }

tests/Elastic.Markdown.Tests/Directives/DirectiveBaseTests.cs

@@ -51,6 +51,8 @@ public abstract class DirectiveTest : IAsyncLifetime
 	protected MarkdownDocument Document { get; private set; }
 	protected MockFileSystem FileSystem { get; }
 	protected TestDiagnosticsCollector Collector { get; }
+	protected DocumentationSet Set { get; set; }
+
 
 	protected DirectiveTest(ITestOutputHelper output, [LanguageInjection("markdown")]string content)
 	{
@@ -62,23 +64,28 @@ protected DirectiveTest(ITestOutputHelper output, [LanguageInjection("markdown")
 		{
 			CurrentDirectory = Paths.Root.FullName
 		});
+		// ReSharper disable once VirtualMemberCallInConstructor
+		// nasty but sub implementations won't use class state.
+		AddToFileSystem(FileSystem);
+
+		var root = FileSystem.DirectoryInfo.New(Path.Combine(Paths.Root.FullName, "docs/source"));
+		FileSystem.GenerateDocSetYaml(root);
 
-		var file = FileSystem.FileInfo.New("docs/source/index.md");
-		var root = file.Directory!;
 		Collector = new TestDiagnosticsCollector(logger);
 		var context = new BuildContext
 		{
 			ReadFileSystem = FileSystem,
 			WriteFileSystem = FileSystem,
 			Collector = Collector
 		};
-		var parser = new MarkdownParser(root, context);
-
-		File = new MarkdownFile(file, root, parser, context);
+		Set = new DocumentationSet(null, null, context);
+		File = Set.GetMarkdownFile("index.md") ?? throw new NullReferenceException();
 		Html = default!; //assigned later
 		Document = default!;
 	}
 
+	protected virtual void AddToFileSystem(MockFileSystem fileSystem) { }
+
 	public virtual async Task InitializeAsync()
 	{
 		var collectTask =  Task.Run(async () => await Collector.StartAsync(default), default);

tests/Elastic.Markdown.Tests/Directives/ImageTests.cs

@@ -2,6 +2,7 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using System.IO.Abstractions.TestingHelpers;
 using Elastic.Markdown.Diagnostics;
 using Elastic.Markdown.Myst.Directives;
 using FluentAssertions;
@@ -18,11 +19,8 @@ public class ImageBlockTests(ITestOutputHelper output) : DirectiveTest<ImageBloc
 """
 )
 {
-	public override Task InitializeAsync()
-	{
-		FileSystem.AddFile(@"docs/source/img/observability.png", "");
-		return base.InitializeAsync();
-	}
+	protected override void AddToFileSystem(MockFileSystem fileSystem) =>
+		fileSystem.AddFile(@"docs/source/img/observability.png", "");
 
 	[Fact]
 	public void ParsesBlock() => Block.Should().NotBeNull();

tests/Elastic.Markdown.Tests/FileInclusion/IncludeTests.cs

@@ -2,6 +2,7 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using System.IO.Abstractions.TestingHelpers;
 using Elastic.Markdown.Diagnostics;
 using Elastic.Markdown.Myst.Directives;
 using Elastic.Markdown.Tests.Directives;
@@ -18,12 +19,11 @@ public class IncludeTests(ITestOutputHelper output) : DirectiveTest<IncludeBlock
 """
 )
 {
-	public override Task InitializeAsync()
+	protected override void AddToFileSystem(MockFileSystem fileSystem)
 	{
 		// language=markdown
 		var inclusion = "*Hello world*";
-		FileSystem.AddFile(@"docs/source/_snippets/test.md", inclusion);
-		return base.InitializeAsync();
+		fileSystem.AddFile(@"docs/source/_snippets/test.md", inclusion);
 	}
 
 	[Fact]
@@ -50,12 +50,11 @@ public class IncludeSubstitutionTests(ITestOutputHelper output) : DirectiveTest<
 """
 )
 {
-	public override Task InitializeAsync()
+	protected override void AddToFileSystem(MockFileSystem fileSystem)
 	{
 		// language=markdown
 		var inclusion = "*Hello {{foo}}*";
-		FileSystem.AddFile(@"docs/source/_snippets/test.md", inclusion);
-		return base.InitializeAsync();
+		fileSystem.AddFile(@"docs/source/_snippets/test.md", inclusion);
 	}
 
 	[Fact]

tests/Elastic.Markdown.Tests/FileInclusion/LiteralIncludeTests.cs

@@ -1,6 +1,8 @@
 // Licensed to Elasticsearch B.V under one or more agreements.
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
+
+using System.IO.Abstractions.TestingHelpers;
 using Elastic.Markdown.Myst.Directives;
 using Elastic.Markdown.Tests.Directives;
 using FluentAssertions;
@@ -17,12 +19,11 @@ public class LiteralIncludeUsingPropertyTests(ITestOutputHelper output) : Direct
 """
 )
 {
-	public override Task InitializeAsync()
+	protected override void AddToFileSystem(MockFileSystem fileSystem)
 	{
 		// language=markdown
 		var inclusion = "*Hello world*";
-		FileSystem.AddFile(@"docs/source/_snippets/test.txt", inclusion);
-		return base.InitializeAsync();
+		fileSystem.AddFile(@"docs/source/_snippets/test.txt", inclusion);
 	}
 
 	[Fact]
@@ -43,12 +44,11 @@ public class LiteralIncludeTests(ITestOutputHelper output) : DirectiveTest<Inclu
 """
 )
 {
-	public override Task InitializeAsync()
+	protected override void AddToFileSystem(MockFileSystem fileSystem)
 	{
 		// language=markdown
 		var inclusion = "*Hello world*";
-		FileSystem.AddFile(@"docs/source/_snippets/test.md", inclusion);
-		return base.InitializeAsync();
+		fileSystem.AddFile(@"docs/source/_snippets/test.md", inclusion);
 	}
 
 	[Fact]