diff --git a/docs/source/docset.yml b/docs/source/docset.yml index cf78b27c1..7ff942688 100644 --- a/docs/source/docset.yml +++ b/docs/source/docset.yml @@ -1,8 +1,25 @@ project: 'doc-builder' +# docs-builder will warn for links to external hosts not declared here +external_hosts: + - slack.com + - mystmd.org + - microsoft.com + - azure.com + - mistral.ai + - amazon.com + - python.org + - cohere.com + - docker.com + - langchain.com + - nodejs.org + - yarnpkg.com + - react.dev + - palletsprojects.com exclude: - '_*.md' toc: - file: index.md + - folder: markup - folder: elastic children: - file: index.md @@ -24,7 +41,6 @@ toc: children: - file: search/req.md - file: search/setup.md - - folder: markup - folder: nested children: - folder: content diff --git a/docs/source/elastic/search-labs/chat.md b/docs/source/elastic/search-labs/chat.md index 699d81e23..0095524b4 100644 --- a/docs/source/elastic/search-labs/chat.md +++ b/docs/source/elastic/search-labs/chat.md @@ -4,8 +4,6 @@ title: Chatbot Tutorial In this tutorial you are going to build a large language model (LLM) chatbot that uses a pattern known as [Retrieval-Augmented Generation (RAG)](https://www.elastic.co/what-is/retrieval-augmented-generation). - - Chatbots built with RAG can overcome some of the limitations that general-purpose conversational models such as ChatGPT have. In particular, they are able to discuss and answer questions about: - Information that is private to your organization. diff --git a/docs/source/elastic/search-labs/install/cloud.md b/docs/source/elastic/search-labs/install/cloud.md index 56a6fea81..368b75e92 100644 --- a/docs/source/elastic/search-labs/install/cloud.md +++ b/docs/source/elastic/search-labs/install/cloud.md @@ -30,14 +30,10 @@ Follow these steps to obtain your Cloud ID: 1. Locate your deployment, and click the **Manage** link under the **Actions** column. 1. The Cloud ID is displayed on the right side of the page. See the screenshot below as a reference. - - ## Creating an API Key For security purposes, it is recommended that you create an API Key to use when authenticating to the Elasticsearch service. - - Follow these steps to create an API Key: 1. Navigate to your [Elastic Cloud](https://cloud.elastic.co/home) home page. diff --git a/docs/source/elastic/search-labs/search/setup.md b/docs/source/elastic/search-labs/search/setup.md index 3bfb77754..9afdc6126 100644 --- a/docs/source/elastic/search-labs/search/setup.md +++ b/docs/source/elastic/search-labs/search/setup.md @@ -12,8 +12,6 @@ Download the starter search application by clicking on the link below. Find a suitable parent directory for your project, such as your *Documents* directory, and extract the contents of the zip file there. This should add a *search-tutorial* directory with several sub-directories and files inside. - - ## Install the Python Dependencies @@ -69,8 +67,6 @@ flask run To confirm that the application is running, open your browser and navigate to http://localhost:5001. - - ```{note} The application in this early stage is just an empty shell. You can type something in the search box and request a search if you like, but the response is always going to be that there are no results. In the following sections you will learn how to load some content in an Elasticsearch index and perform searches. ``` diff --git a/docs/source/index.md b/docs/source/index.md index f4feea351..2156a72c8 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -4,6 +4,8 @@ title: Elastic Docs v3 Elastic Docs v3 is built with +Test during zoom with colleen + TODO ADD README for doc-builder :::{tip} diff --git a/docs/source/markup/admonitions.md b/docs/source/markup/admonitions.md index 2bd14ce5a..0184c4a7b 100644 --- a/docs/source/markup/admonitions.md +++ b/docs/source/markup/admonitions.md @@ -6,39 +6,15 @@ Admonitions bring the attention of readers. ## Basic admonitions -```{attention} -:name: attention_ref -This is an 'attention' admonition -``` - ```{caution} +:name: caution_ref This is a 'caution' admonition ``` -```{danger} -This is a 'danger' admonition -``` - -```{error} -This is an 'error' admonition -``` - -```{hint} -This is an 'hint' admonition -``` - -```{important} -This is an 'important' admonition -``` - ```{note} This is a 'note' admonition ``` -```{seealso} -This is a 'seealso' admonition -``` - ```{tip} This is a tip ``` @@ -84,4 +60,4 @@ Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor i ## Link to admonitions You can add a 'name' option to an admonition, so that you can link to it elsewhere -Here is a [link to attention](#attention_ref) +Here is a [link to attention](#caution_ref) diff --git a/src/Elastic.Markdown/BuildContext.cs b/src/Elastic.Markdown/BuildContext.cs index 54d727fef..6d73b608e 100644 --- a/src/Elastic.Markdown/BuildContext.cs +++ b/src/Elastic.Markdown/BuildContext.cs @@ -3,13 +3,20 @@ // See the LICENSE file in the project root for more information using System.IO.Abstractions; using Elastic.Markdown.Diagnostics; +using Elastic.Markdown.IO; namespace Elastic.Markdown; public record BuildContext { - public required IFileSystem ReadFileSystem { get; init; } - public required IFileSystem WriteFileSystem { get; init; } + public IFileSystem ReadFileSystem { get; } + public IFileSystem WriteFileSystem { get; } + + public IDirectoryInfo SourcePath { get; } + public IDirectoryInfo OutputPath { get; } + + public IFileInfo ConfigurationPath { get; } + public required DiagnosticsCollector Collector { get; init; } public bool Force { get; init; } @@ -22,4 +29,32 @@ public string? UrlPathPrefix private readonly string? _urlPathPrefix; + public BuildContext(IFileSystem fileSystem) + : this(fileSystem, fileSystem, null, null) { } + + public BuildContext(IFileSystem readFileSystem, IFileSystem writeFileSystem) + : this(readFileSystem, writeFileSystem, null, null) { } + + public BuildContext(IFileSystem readFileSystem, IFileSystem writeFileSystem, string? source, string? output) + { + ReadFileSystem = readFileSystem; + WriteFileSystem = writeFileSystem; + + SourcePath = !string.IsNullOrWhiteSpace(source) + ? ReadFileSystem.DirectoryInfo.New(source) + : ReadFileSystem.DirectoryInfo.New(Path.Combine(Paths.Root.FullName, "docs/source")); + OutputPath = !string.IsNullOrWhiteSpace(output) + ? WriteFileSystem.DirectoryInfo.New(output) + : WriteFileSystem.DirectoryInfo.New(Path.Combine(Paths.Root.FullName, ".artifacts/docs/html")); + + ConfigurationPath = + SourcePath.EnumerateFiles("docset.yml", SearchOption.AllDirectories).FirstOrDefault() + ?? ReadFileSystem.FileInfo.New(Path.Combine(SourcePath.FullName, "docset.yml")); + + if (ConfigurationPath.FullName != SourcePath.FullName) + SourcePath = ConfigurationPath.Directory!; + + + } + } diff --git a/src/Elastic.Markdown/DocumentationGenerator.cs b/src/Elastic.Markdown/DocumentationGenerator.cs index 57fed5b66..bc6e74e61 100644 --- a/src/Elastic.Markdown/DocumentationGenerator.cs +++ b/src/Elastic.Markdown/DocumentationGenerator.cs @@ -33,16 +33,15 @@ public class DocumentationGenerator public DocumentationGenerator( DocumentationSet docSet, - BuildContext context, ILoggerFactory logger ) { - _readFileSystem = context.ReadFileSystem; - _writeFileSystem = context.WriteFileSystem; + _readFileSystem = docSet.Context.ReadFileSystem; + _writeFileSystem = docSet.Context.WriteFileSystem; _logger = logger.CreateLogger(nameof(DocumentationGenerator)); DocumentationSet = docSet; - Context = context; + Context = docSet.Context; HtmlWriter = new HtmlWriter(DocumentationSet, _writeFileSystem); _logger.LogInformation($"Created documentation set for: {DocumentationSet.Name}"); @@ -50,19 +49,6 @@ ILoggerFactory logger _logger.LogInformation($"Output directory: {docSet.OutputPath} Exists: {docSet.OutputPath.Exists}"); } - public static DocumentationGenerator Create( - string? path, - string? output, - BuildContext context, - ILoggerFactory logger - ) - { - var sourcePath = path != null ? context.ReadFileSystem.DirectoryInfo.New(path) : null; - var outputPath = output != null ? context.WriteFileSystem.DirectoryInfo.New(output) : null; - var docSet = new DocumentationSet(sourcePath, outputPath, context); - return new DocumentationGenerator(docSet, context, logger); - } - public OutputState? OutputState { get diff --git a/src/Elastic.Markdown/IO/ConfigurationFile.cs b/src/Elastic.Markdown/IO/ConfigurationFile.cs index 5c7002ce0..1543470dd 100644 --- a/src/Elastic.Markdown/IO/ConfigurationFile.cs +++ b/src/Elastic.Markdown/IO/ConfigurationFile.cs @@ -23,6 +23,11 @@ public record ConfigurationFile : DocumentationFile public HashSet Files { get; } = new(StringComparer.OrdinalIgnoreCase); public HashSet ImplicitFolders { get; } = new(StringComparer.OrdinalIgnoreCase); public Glob[] Globs { get; } = []; + public HashSet ExternalLinkHosts { get; } = new(StringComparer.OrdinalIgnoreCase) + { + "elastic.co", + "github.com", + }; public ConfigurationFile(IFileInfo sourceFile, IDirectoryInfo rootPath, BuildContext context) : base(sourceFile, rootPath) @@ -62,6 +67,12 @@ public ConfigurationFile(IFileInfo sourceFile, IDirectoryInfo rootPath, BuildCon .Select(Glob.Parse) .ToArray(); break; + case "external_hosts": + var hosts = ReadStringArray(entry) + .ToArray(); + foreach (var host in hosts) + ExternalLinkHosts.Add(host); + break; case "toc": var entries = ReadChildren(entry, string.Empty); diff --git a/src/Elastic.Markdown/IO/DocumentationSet.cs b/src/Elastic.Markdown/IO/DocumentationSet.cs index 1662c664c..651bd8eeb 100644 --- a/src/Elastic.Markdown/IO/DocumentationSet.cs +++ b/src/Elastic.Markdown/IO/DocumentationSet.cs @@ -10,10 +10,12 @@ namespace Elastic.Markdown.IO; public class DocumentationSet { + public BuildContext Context { get; } public string Name { get; } + public IFileInfo OutputStateFile { get; } + public IDirectoryInfo SourcePath { get; } public IDirectoryInfo OutputPath { get; } - public IFileInfo OutputStateFile { get; } public DateTimeOffset LastWrite { get; } @@ -21,30 +23,18 @@ public class DocumentationSet public MarkdownParser MarkdownParser { get; } - public DocumentationSet(BuildContext context) : this(null, null, context) { } - - public DocumentationSet(IDirectoryInfo? sourcePath, IDirectoryInfo? outputPath, BuildContext context) + public DocumentationSet(BuildContext context) { - SourcePath = sourcePath ?? context.ReadFileSystem.DirectoryInfo.New(Path.Combine(Paths.Root.FullName, "docs/source")); - OutputPath = outputPath ?? context.WriteFileSystem.DirectoryInfo.New(Path.Combine(Paths.Root.FullName, ".artifacts/docs/html")); + Context = context; + SourcePath = context.SourcePath; + OutputPath = context.OutputPath; + Configuration = new ConfigurationFile(context.ConfigurationPath, SourcePath, context); - var configurationFile = SourcePath.EnumerateFiles("docset.yml", SearchOption.AllDirectories).FirstOrDefault(); - if (configurationFile is null) - { - configurationFile = context.ReadFileSystem.FileInfo.New(Path.Combine(SourcePath.FullName, "docset.yml")); - context.EmitWarning(configurationFile, "No configuration file found"); - } - - if (configurationFile.Directory!.FullName != SourcePath.FullName) - SourcePath = configurationFile.Directory; - - MarkdownParser = new MarkdownParser(SourcePath, context, GetTitle); + MarkdownParser = new MarkdownParser(SourcePath, context, GetMarkdownFile, Configuration); Name = SourcePath.FullName; OutputStateFile = OutputPath.FileSystem.FileInfo.New(Path.Combine(OutputPath.FullName, ".doc.state")); - Configuration = new ConfigurationFile(configurationFile, SourcePath, context); - Files = context.ReadFileSystem.Directory .EnumerateFiles(SourcePath.FullName, "*.*", SearchOption.AllDirectories) .Select(f => context.ReadFileSystem.FileInfo.New(f)) @@ -68,10 +58,9 @@ public DocumentationSet(IDirectoryInfo? sourcePath, IDirectoryInfo? outputPath, Tree = new DocumentationFolder(Configuration.TableOfContents, FlatMappedFiles, folderFiles); } - private string? GetTitle(string relativePath) => GetMarkdownFile(relativePath)?.YamlFrontMatter?.Title; - - public MarkdownFile? GetMarkdownFile(string relativePath) + public MarkdownFile? GetMarkdownFile(IFileInfo sourceFile) { + var relativePath = Path.GetRelativePath(SourcePath.FullName, sourceFile.FullName); if (FlatMappedFiles.TryGetValue(relativePath, out var file) && file is MarkdownFile markdownFile) return markdownFile; return null; diff --git a/src/Elastic.Markdown/IO/MarkdownFile.cs b/src/Elastic.Markdown/IO/MarkdownFile.cs index 1d7ec43cf..87333ae7f 100644 --- a/src/Elastic.Markdown/IO/MarkdownFile.cs +++ b/src/Elastic.Markdown/IO/MarkdownFile.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information using System.IO.Abstractions; using Elastic.Markdown.Myst; +using Elastic.Markdown.Myst.Directives; using Elastic.Markdown.Slices; using Markdig; using Markdig.Extensions.Yaml; @@ -34,8 +35,12 @@ public string? NavigationTitle private set => _navigationTitle = value; } - private readonly List _tableOfContent = new(); - public IReadOnlyCollection TableOfContents => _tableOfContent; + //indexed by slug + private readonly Dictionary _tableOfContent = new(); + public IReadOnlyDictionary TableOfContents => _tableOfContent; + + private readonly HashSet _additionalLabels = new(); + public IReadOnlySet AdditionalLabels => _additionalLabels; public string FileName { get; } public string Url => $"{UrlPathPrefix}/{RelativePath.Replace(".md", ".html")}"; @@ -76,7 +81,20 @@ private void ReadDocumentInstructions(MarkdownDocument document) .Select(title => new PageTocItem { Heading = title!, Slug = _slugHelper.GenerateSlug(title) }) .ToList(); _tableOfContent.Clear(); - _tableOfContent.AddRange(contents); + foreach (var t in contents) + _tableOfContent[t.Slug] = t; + + var labels = document.Descendants() + .Select(b=>b.CrossReferenceName) + .Where(l=>!string.IsNullOrWhiteSpace(l)) + .Select(_slugHelper.GenerateSlug) + .ToArray(); + foreach(var label in labels) + { + if (!string.IsNullOrEmpty(label)) + _additionalLabels.Add(label); + } + _instructionsParsed = true; } diff --git a/src/Elastic.Markdown/Myst/Directives/AdmonitionBlock.cs b/src/Elastic.Markdown/Myst/Directives/AdmonitionBlock.cs index 3f6da6613..39f43e078 100644 --- a/src/Elastic.Markdown/Myst/Directives/AdmonitionBlock.cs +++ b/src/Elastic.Markdown/Myst/Directives/AdmonitionBlock.cs @@ -10,9 +10,7 @@ public class AdmonitionBlock(DirectiveBlockParser parser, string admonition, Dic public override string Directive => Admonition; - public string? Label { get; protected set; } public string? Classes { get; protected set; } - public string? CrossReferenceName { get; private set; } public bool? DropdownOpen { get; private set; } public string Title @@ -31,7 +29,6 @@ public string Title public override void FinalizeAndValidate(ParserContext context) { - Label = Prop("label"); Classes = Properties.GetValueOrDefault("class"); CrossReferenceName = Properties.GetValueOrDefault("name"); DropdownOpen = PropBool("open"); diff --git a/src/Elastic.Markdown/Myst/Directives/CodeBlock.cs b/src/Elastic.Markdown/Myst/Directives/CodeBlock.cs index 306c0ea7f..3d6be1474 100644 --- a/src/Elastic.Markdown/Myst/Directives/CodeBlock.cs +++ b/src/Elastic.Markdown/Myst/Directives/CodeBlock.cs @@ -8,7 +8,6 @@ public class CodeBlock(DirectiveBlockParser parser, string directive, Dictionary { public override string Directive => directive; public string? Caption { get; private set; } - public string? CrossReferenceName { get; private set; } public string Language { @@ -23,6 +22,6 @@ public string Language public override void FinalizeAndValidate(ParserContext context) { Caption = Properties.GetValueOrDefault("caption"); - CrossReferenceName = Properties.GetValueOrDefault("name"); + CrossReferenceName = Prop("name", "label"); } } diff --git a/src/Elastic.Markdown/Myst/Directives/DirectiveBlock.cs b/src/Elastic.Markdown/Myst/Directives/DirectiveBlock.cs index bb418951b..d18e0b453 100644 --- a/src/Elastic.Markdown/Myst/Directives/DirectiveBlock.cs +++ b/src/Elastic.Markdown/Myst/Directives/DirectiveBlock.cs @@ -27,6 +27,8 @@ public abstract class DirectiveBlock(DirectiveBlockParser parser, Dictionary Properties { get; } = properties; + public string? CrossReferenceName { get; protected set; } + /// public char FencedChar { get; set; } diff --git a/src/Elastic.Markdown/Myst/Directives/DirectiveBlockParser.cs b/src/Elastic.Markdown/Myst/Directives/DirectiveBlockParser.cs index a200533c1..6266add74 100644 --- a/src/Elastic.Markdown/Myst/Directives/DirectiveBlockParser.cs +++ b/src/Elastic.Markdown/Myst/Directives/DirectiveBlockParser.cs @@ -58,7 +58,6 @@ public DirectiveBlockParser() { "code-cell", 8 }, { "admonition", 3 }, - { "attention", 3 }, { "danger", 3 }, { "error", 3 }, { "hint", 3 }, diff --git a/src/Elastic.Markdown/Myst/Directives/DirectiveHtmlRenderer.cs b/src/Elastic.Markdown/Myst/Directives/DirectiveHtmlRenderer.cs index 2e2e9031c..f0a61a8c3 100644 --- a/src/Elastic.Markdown/Myst/Directives/DirectiveHtmlRenderer.cs +++ b/src/Elastic.Markdown/Myst/Directives/DirectiveHtmlRenderer.cs @@ -208,7 +208,7 @@ private void WriteIncludeBlock(HtmlRenderer renderer, IncludeBlock block) if (!block.Found || block.IncludePath is null) return; - var parser = new MarkdownParser(block.DocumentationSourcePath, block.Build, block.GetTitle); + var parser = new MarkdownParser(block.DocumentationSourcePath, block.Build, block.GetMarkdownFile, block.Configuration); var file = block.FileSystem.FileInfo.New(block.IncludePath); var document = parser.ParseAsync(file, block.FrontMatter, default).GetAwaiter().GetResult(); var html = document.ToHtml(parser.Pipeline); diff --git a/src/Elastic.Markdown/Myst/Directives/ImageBlock.cs b/src/Elastic.Markdown/Myst/Directives/ImageBlock.cs index e14b50308..ba4651c3b 100644 --- a/src/Elastic.Markdown/Myst/Directives/ImageBlock.cs +++ b/src/Elastic.Markdown/Myst/Directives/ImageBlock.cs @@ -46,15 +46,11 @@ public class ImageBlock(DirectiveBlockParser parser, Dictionary /// public string? Target { get; set; } - /// - /// A reference target for the admonition (see cross-referencing). - /// - public string? Label { get; private set; } - public string? ImageUrl { get; private set; } public bool Found { get; private set; } + public string? Label { get; private set; } public override void FinalizeAndValidate(ParserContext context) { diff --git a/src/Elastic.Markdown/Myst/Directives/IncludeBlock.cs b/src/Elastic.Markdown/Myst/Directives/IncludeBlock.cs index 5a387ed87..974fb323d 100644 --- a/src/Elastic.Markdown/Myst/Directives/IncludeBlock.cs +++ b/src/Elastic.Markdown/Myst/Directives/IncludeBlock.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information using System.IO.Abstractions; using Elastic.Markdown.Diagnostics; +using Elastic.Markdown.IO; namespace Elastic.Markdown.Myst.Directives; @@ -13,7 +14,9 @@ public class IncludeBlock(DirectiveBlockParser parser, Dictionary? GetTitle { get; } = context.GetTitle; + public Func? GetMarkdownFile { get; } = context.GetMarkdownFile; + + public ConfigurationFile Configuration { get; } = context.Configuration; public IFileSystem FileSystem { get; } = context.Build.ReadFileSystem; diff --git a/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs b/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs index 4f129183b..a2cebb1ad 100644 --- a/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs +++ b/src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs @@ -57,7 +57,9 @@ public override bool Match(InlineProcessor processor, ref StringSlice slice) if (Uri.TryCreate(url, UriKind.Absolute, out var uri) && uri.Scheme.StartsWith("http")) { - processor.EmitWarning(line, column, length, $"external URI: {uri} "); + var baseDomain = string.Join('.', uri.Host.Split('.')[^2..]); + if (!context.Configuration.ExternalLinkHosts.Contains(baseDomain)) + processor.EmitWarning(line, column, length, $"external URI: {uri} "); return match; } @@ -65,20 +67,47 @@ public override bool Match(InlineProcessor processor, ref StringSlice slice) if (url.StartsWith('/')) includeFrom = context.Parser.SourcePath.FullName; - var pathOnDisk = Path.Combine(includeFrom, url.TrimStart('/')); - if (!context.Build.ReadFileSystem.File.Exists(pathOnDisk)) - processor.EmitError(line, column, length, $"`{url}` does not exist. resolved to `{pathOnDisk}"); + var anchors = url.Split('#'); + var anchor = anchors.Length > 1 ? anchors[1] : null; + url = anchors[0]; - if (link.FirstChild == null) + if (!string.IsNullOrWhiteSpace(url)) { - var title = context.GetTitle?.Invoke(url); - if (!string.IsNullOrEmpty(title)) + var pathOnDisk = Path.Combine(includeFrom, url.TrimStart('/')); + if (!context.Build.ReadFileSystem.File.Exists(pathOnDisk)) + processor.EmitError(line, column, length, $"`{url}` does not exist. resolved to `{pathOnDisk}"); + } + else + link.Url = ""; + + if (link.FirstChild == null || !string.IsNullOrEmpty(anchor)) + { + var file = string.IsNullOrWhiteSpace(url) ? context.Path + : context.Build.ReadFileSystem.FileInfo.New(Path.Combine(context.Build.SourcePath.FullName, url)); + var markdown = context.GetMarkdownFile?.Invoke(file); + var title = markdown?.Title; + + if (!string.IsNullOrEmpty(anchor)) + { + if (markdown == null || (!markdown.TableOfContents.TryGetValue(anchor, out var heading) + && !markdown.AdditionalLabels.Contains(anchor))) + processor.EmitError(line, column, length, $"`{anchor}` does not exist in {markdown?.FileName}."); + + else if (link.FirstChild == null && heading != null) + title += " > " + heading.Heading; + + } + + if (link.FirstChild == null && !string.IsNullOrEmpty(title)) link.AppendChild(new LiteralInline(title)); } if (url.EndsWith(".md")) link.Url = Path.ChangeExtension(url, ".html"); + if (!string.IsNullOrEmpty(anchor)) + link.Url += $"#{anchor}"; + return match; diff --git a/src/Elastic.Markdown/Myst/MarkdownParser.cs b/src/Elastic.Markdown/Myst/MarkdownParser.cs index b57a371e7..925af0d13 100644 --- a/src/Elastic.Markdown/Myst/MarkdownParser.cs +++ b/src/Elastic.Markdown/Myst/MarkdownParser.cs @@ -4,6 +4,7 @@ using System.IO.Abstractions; using Cysharp.IO; +using Elastic.Markdown.IO; using Elastic.Markdown.Myst.Comments; using Elastic.Markdown.Myst.Directives; using Elastic.Markdown.Myst.InlineParsers; @@ -14,16 +15,22 @@ namespace Elastic.Markdown.Myst; -public class MarkdownParser(IDirectoryInfo sourcePath, BuildContext context, Func? getTitle) +public class MarkdownParser( + IDirectoryInfo sourcePath, + BuildContext context, + Func? getMarkdownFile, + ConfigurationFile configuration) { public IDirectoryInfo SourcePath { get; } = sourcePath; public BuildContext Context { get; } = context; - public MarkdownPipeline MinimalPipeline { get; } = + //TODO directive properties are stateful, rewrite this so we can cache builders + public MarkdownPipeline MinimalPipeline => new MarkdownPipelineBuilder() .UseDiagnosticLinks() - .UseSubstitution() .UseYamlFrontMatter() + .UseDirectives() + .UseSubstitution() .Build(); public MarkdownPipeline Pipeline => @@ -46,19 +53,19 @@ public class MarkdownParser(IDirectoryInfo sourcePath, BuildContext context, Fun public Task MinimalParseAsync(IFileInfo path, Cancel ctx) { - var context = new ParserContext(this, path, null, Context) + var context = new ParserContext(this, path, null, Context, configuration) { SkipValidation = true, - GetTitle = getTitle + GetMarkdownFile = getMarkdownFile }; return ParseAsync(path, context, MinimalPipeline, ctx); } public Task ParseAsync(IFileInfo path, YamlFrontMatter? matter, Cancel ctx) { - var context = new ParserContext(this, path, matter, Context) + var context = new ParserContext(this, path, matter, Context, configuration) { - GetTitle = getTitle + GetMarkdownFile = getMarkdownFile }; return ParseAsync(path, context, Pipeline, ctx); } diff --git a/src/Elastic.Markdown/Myst/ParserContext.cs b/src/Elastic.Markdown/Myst/ParserContext.cs index 0b071193b..d4a0067e0 100644 --- a/src/Elastic.Markdown/Myst/ParserContext.cs +++ b/src/Elastic.Markdown/Myst/ParserContext.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information using System.IO.Abstractions; +using Elastic.Markdown.IO; using Markdig; using Markdig.Parsers; @@ -30,12 +31,14 @@ public class ParserContext : MarkdownParserContext public ParserContext(MarkdownParser markdownParser, IFileInfo path, YamlFrontMatter? frontMatter, - BuildContext context) + BuildContext context, + ConfigurationFile configuration) { Parser = markdownParser; Path = path; FrontMatter = frontMatter; Build = context; + Configuration = configuration; if (frontMatter?.Properties is { } props) { @@ -47,10 +50,11 @@ public ParserContext(MarkdownParser markdownParser, Properties["page_title"] = title; } + public ConfigurationFile Configuration { get; } public MarkdownParser Parser { get; } public IFileInfo Path { get; } public YamlFrontMatter? FrontMatter { get; } public BuildContext Build { get; } public bool SkipValidation { get; init; } - public Func? GetTitle { get; init; } + public Func? GetMarkdownFile { get; init; } } diff --git a/src/Elastic.Markdown/Slices/HtmlWriter.cs b/src/Elastic.Markdown/Slices/HtmlWriter.cs index 462959c0c..9706dd4a5 100644 --- a/src/Elastic.Markdown/Slices/HtmlWriter.cs +++ b/src/Elastic.Markdown/Slices/HtmlWriter.cs @@ -48,7 +48,7 @@ public async Task RenderLayout(MarkdownFile markdown, Cancel ctx = defau { Title = markdown.Title ?? "[TITLE NOT SET]", MarkdownHtml = html, - PageTocItems = markdown.TableOfContents, + PageTocItems = markdown.TableOfContents.Values.ToList(), Tree = DocumentationSet.Tree, CurrentDocument = markdown, NavigationHtml = navigationHtml, diff --git a/src/docs-builder/Cli/Commands.cs b/src/docs-builder/Cli/Commands.cs index ae00121a5..1ad119d6d 100644 --- a/src/docs-builder/Cli/Commands.cs +++ b/src/docs-builder/Cli/Commands.cs @@ -8,6 +8,7 @@ using Documentation.Builder.Http; using Elastic.Markdown; using Elastic.Markdown.Diagnostics; +using Elastic.Markdown.IO; using Microsoft.Extensions.Logging; namespace Documentation.Builder.Cli; @@ -50,15 +51,14 @@ public async Task Generate( { pathPrefix ??= githubActionsService.GetInput("prefix"); var fileSystem = new FileSystem(); - var context = new BuildContext + var context = new BuildContext(fileSystem, fileSystem, path, output) { UrlPathPrefix = pathPrefix, Force = force ?? false, - ReadFileSystem = fileSystem, - WriteFileSystem = fileSystem, Collector = new ConsoleDiagnosticsCollector(logger, githubActionsService) }; - var generator = DocumentationGenerator.Create(path, output, context, logger); + var set = new DocumentationSet(context); + var generator = new DocumentationGenerator(set, logger); await generator.GenerateAll(ctx); return context.Collector.Errors > 1 ? 1 : 0; } diff --git a/src/docs-builder/Diagnostics/ErrorCollector.cs b/src/docs-builder/Diagnostics/ErrorCollector.cs index a4b4f5eee..113f7d0e7 100644 --- a/src/docs-builder/Diagnostics/ErrorCollector.cs +++ b/src/docs-builder/Diagnostics/ErrorCollector.cs @@ -83,6 +83,9 @@ public override async Task StopAsync(Cancel ctx) // Render the report report.Render(AnsiConsole.Console); AnsiConsole.WriteLine(); + AnsiConsole.Write(new Markup($" [bold red]{Errors} Errors[/] / [bold blue]{Warnings} Warnings[/]")); + AnsiConsole.WriteLine(); + AnsiConsole.WriteLine(); await Task.CompletedTask; } } diff --git a/src/docs-builder/Http/DocumentationWebHost.cs b/src/docs-builder/Http/DocumentationWebHost.cs index 7b5f8c773..889d6549f 100644 --- a/src/docs-builder/Http/DocumentationWebHost.cs +++ b/src/docs-builder/Http/DocumentationWebHost.cs @@ -26,10 +26,8 @@ public DocumentationWebHost(string? path, ILoggerFactory logger, IFileSystem fil { var builder = WebApplication.CreateSlimBuilder(); var sourcePath = path != null ? fileSystem.DirectoryInfo.New(path) : null; - var context = new BuildContext + var context = new BuildContext(fileSystem) { - ReadFileSystem = fileSystem, - WriteFileSystem = fileSystem, Collector = new ConsoleDiagnosticsCollector(logger) }; builder.Services.AddSingleton(_ => new ReloadableGeneratorState(sourcePath, null, context, logger)); diff --git a/src/docs-builder/Http/ReloadableGeneratorState.cs b/src/docs-builder/Http/ReloadableGeneratorState.cs index 8722a6dd6..38de0b06b 100644 --- a/src/docs-builder/Http/ReloadableGeneratorState.cs +++ b/src/docs-builder/Http/ReloadableGeneratorState.cs @@ -20,15 +20,15 @@ ILoggerFactory logger private IDirectoryInfo? SourcePath { get; } = sourcePath; private IDirectoryInfo? OutputPath { get; } = outputPath; - private DocumentationGenerator _generator = new(new DocumentationSet(sourcePath, outputPath, context), context, logger); + private DocumentationGenerator _generator = new(new DocumentationSet(context), logger); public DocumentationGenerator Generator => _generator; public async Task ReloadAsync(Cancel ctx) { SourcePath?.Refresh(); OutputPath?.Refresh(); - var docSet = new DocumentationSet(SourcePath, OutputPath, context); - var generator = new DocumentationGenerator(docSet, context, logger); + var docSet = new DocumentationSet(context); + var generator = new DocumentationGenerator(docSet, logger); await generator.ResolveDirectoryTree(ctx); Interlocked.Exchange(ref _generator, generator); } diff --git a/tests/Elastic.Markdown.Tests/Directives/DirectiveBaseTests.cs b/tests/Elastic.Markdown.Tests/Directives/DirectiveBaseTests.cs index db32ff52c..7fcd32118 100644 --- a/tests/Elastic.Markdown.Tests/Directives/DirectiveBaseTests.cs +++ b/tests/Elastic.Markdown.Tests/Directives/DirectiveBaseTests.cs @@ -72,14 +72,12 @@ protected DirectiveTest(ITestOutputHelper output, [LanguageInjection("markdown") FileSystem.GenerateDocSetYaml(root); Collector = new TestDiagnosticsCollector(logger); - var context = new BuildContext + var context = new BuildContext(FileSystem) { - ReadFileSystem = FileSystem, - WriteFileSystem = FileSystem, Collector = Collector }; - Set = new DocumentationSet(null, null, context); - File = Set.GetMarkdownFile("index.md") ?? throw new NullReferenceException(); + Set = new DocumentationSet(context); + File = Set.GetMarkdownFile(FileSystem.FileInfo.New("docs/source/index.md")) ?? throw new NullReferenceException(); Html = default!; //assigned later Document = default!; } diff --git a/tests/Elastic.Markdown.Tests/Inline/AnchorLinkTests.cs b/tests/Elastic.Markdown.Tests/Inline/AnchorLinkTests.cs new file mode 100644 index 000000000..85b621ecd --- /dev/null +++ b/tests/Elastic.Markdown.Tests/Inline/AnchorLinkTests.cs @@ -0,0 +1,130 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions.TestingHelpers; +using FluentAssertions; +using JetBrains.Annotations; +using Markdig.Syntax.Inlines; +using Xunit.Abstractions; + +namespace Elastic.Markdown.Tests.Inline; + +public abstract class AnchorLinkTestBase(ITestOutputHelper output, [LanguageInjection("markdown")] string content) + : InlineTest(output, + $""" + ## Hello world + + A paragraph + + {content} + + """) +{ + protected override void AddToFileSystem(MockFileSystem fileSystem) + { + // language=markdown + var inclusion = +""" +--- +title: Special Requirements +--- + +## Sub Requirements + +To follow this tutorial you will need to install the following components: +"""; + fileSystem.AddFile(@"docs/source/elastic/search-labs/search/req.md", inclusion); + fileSystem.AddFile(@"docs/source/_static/img/observability.png", new MockFileData("")); + } + +} + +public class InPageAnchorTests(ITestOutputHelper output) : AnchorLinkTestBase(output, +""" +[Hello](#hello-world) +""" +) +{ + [Fact] + public void GeneratesHtml() => + // language=html + Html.Should().Contain( + """

Hello

""" + ); + + [Fact] + public void HasNoErrors() => Collector.Diagnostics.Should().HaveCount(0); +} + +public class ExternalPageAnchorTests(ITestOutputHelper output) : AnchorLinkTestBase(output, +""" +[Sub Requirements](elastic/search-labs/search/req.md#sub-requirements) +""" +) +{ + [Fact] + public void GeneratesHtml() => + // language=html + Html.Should().Contain( + """

Sub Requirements

""" + ); + + [Fact] + public void HasNoErrors() => Collector.Diagnostics.Should().HaveCount(0); +} + +public class ExternalPageAnchorAutoTitleTests(ITestOutputHelper output) : AnchorLinkTestBase(output, +""" +[](elastic/search-labs/search/req.md#sub-requirements) +""" +) +{ + [Fact] + public void GeneratesHtml() => + // language=html + Html.Should().Contain( + """

Special Requirements > Sub Requirements

""" + ); + + [Fact] + public void HasNoErrors() => Collector.Diagnostics.Should().HaveCount(0); +} + + +public class InPageBadAnchorTests(ITestOutputHelper output) : AnchorLinkTestBase(output, +""" +[Hello](#hello-world2) +""" +) +{ + [Fact] + public void GeneratesHtml() => + // language=html + Html.Should().Contain( + """

Hello

""" + ); + + [Fact] + public void HasError() => Collector.Diagnostics.Should().HaveCount(1) + .And.Contain(d => d.Message.Contains("`hello-world2` does not exist")); +} + +public class ExternalPageBadAnchorTests(ITestOutputHelper output) : AnchorLinkTestBase(output, +""" +[Sub Requirements](elastic/search-labs/search/req.md#sub-requirements2) +""" +) +{ + [Fact] + public void GeneratesHtml() => + // language=html + Html.Should().Contain( + """

Sub Requirements

""" + ); + + [Fact] + public void HasError() => Collector.Diagnostics.Should().HaveCount(1) + .And.Contain(d => d.Message.Contains("`sub-requirements2` does not exist")); +} + diff --git a/tests/Elastic.Markdown.Tests/Inline/DirectiveBlockLinkTests.cs b/tests/Elastic.Markdown.Tests/Inline/DirectiveBlockLinkTests.cs new file mode 100644 index 000000000..05f0d16ea --- /dev/null +++ b/tests/Elastic.Markdown.Tests/Inline/DirectiveBlockLinkTests.cs @@ -0,0 +1,78 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.IO.Abstractions.TestingHelpers; +using FluentAssertions; +using JetBrains.Annotations; +using Markdig.Syntax.Inlines; +using Xunit.Abstractions; + +namespace Elastic.Markdown.Tests.Inline; + +public abstract class DirectiveBlockLinkTests(ITestOutputHelper output, [LanguageInjection("markdown")] string content) + : InlineTest(output, +$$""" +```{caution} +:name: caution_ref +This is a 'caution' admonition +``` + +{{content}} + +""") +{ + protected override void AddToFileSystem(MockFileSystem fileSystem) + { + // language=markdown + var inclusion = +""" +--- +title: Special Requirements +--- + +```{attention} +:name: hint_ref +This is a 'caution' admonition +``` +"""; + fileSystem.AddFile(@"docs/source/elastic/search-labs/search/req.md", inclusion); + fileSystem.AddFile(@"docs/source/_static/img/observability.png", new MockFileData("")); + } + +} + +public class InPageDirectiveLinkTests(ITestOutputHelper output) : DirectiveBlockLinkTests(output, +""" +[Hello](#caution_ref) +""" +) +{ + [Fact] + public void GeneratesHtml() => + // language=html + Html.Should().Contain( + """

Hello

""" + ); + + [Fact] + public void HasNoErrors() => Collector.Diagnostics.Should().HaveCount(0); +} + +public class ExternalDirectiveLinkTests(ITestOutputHelper output) : DirectiveBlockLinkTests(output, +""" +[Sub Requirements](elastic/search-labs/search/req.md#hint_ref) +""" +) +{ + [Fact] + public void GeneratesHtml() => + // language=html + Html.Should().Contain( + """

Sub Requirements

""" + ); + + [Fact] + public void HasNoErrors() => Collector.Diagnostics.Should().HaveCount(0); +} + diff --git a/tests/Elastic.Markdown.Tests/Inline/InlneBaseTests.cs b/tests/Elastic.Markdown.Tests/Inline/InlneBaseTests.cs index 83cbc0137..acd7af3c3 100644 --- a/tests/Elastic.Markdown.Tests/Inline/InlneBaseTests.cs +++ b/tests/Elastic.Markdown.Tests/Inline/InlneBaseTests.cs @@ -46,12 +46,7 @@ public override async Task InitializeAsync() { await base.InitializeAsync(); Block = Document - .Where(block => block is ParagraphBlock) - .Cast() - .FirstOrDefault()? - .Inline? - .Where(block => block is TDirective) - .Cast() + .Descendants() .FirstOrDefault(); } @@ -87,14 +82,12 @@ protected InlineTest(ITestOutputHelper output, [LanguageInjection("markdown")]st FileSystem.GenerateDocSetYaml(root); Collector = new TestDiagnosticsCollector(logger); - var context = new BuildContext + var context = new BuildContext(FileSystem) { - ReadFileSystem = FileSystem, - WriteFileSystem = FileSystem, Collector = Collector }; - Set = new DocumentationSet(null, null, context); - File = Set.GetMarkdownFile("index.md") ?? throw new NullReferenceException(); + Set = new DocumentationSet(context); + File = Set.GetMarkdownFile(FileSystem.FileInfo.New("docs/source/index.md")) ?? throw new NullReferenceException(); Html = default!; //assigned later Document = default!; } diff --git a/tests/Elastic.Markdown.Tests/OutputDirectoryTests.cs b/tests/Elastic.Markdown.Tests/OutputDirectoryTests.cs index 8b1f3f615..281ef0bc1 100644 --- a/tests/Elastic.Markdown.Tests/OutputDirectoryTests.cs +++ b/tests/Elastic.Markdown.Tests/OutputDirectoryTests.cs @@ -23,14 +23,12 @@ public async Task CreatesDefaultOutputDirectory() { CurrentDirectory = Paths.Root.FullName }); - var context = new BuildContext + var context = new BuildContext(fileSystem) { - ReadFileSystem = fileSystem, - WriteFileSystem = fileSystem, Collector = new DiagnosticsCollector(logger, []) }; - var set = new DocumentationSet(null, null, context); - var generator = new DocumentationGenerator(set, context, logger); + var set = new DocumentationSet(context); + var generator = new DocumentationGenerator(set, logger); await generator.GenerateAll(default); diff --git a/tests/Elastic.Markdown.Tests/SiteMap/NavigationTestsBase.cs b/tests/Elastic.Markdown.Tests/SiteMap/NavigationTestsBase.cs index 5f7d850f8..1559d8e27 100644 --- a/tests/Elastic.Markdown.Tests/SiteMap/NavigationTestsBase.cs +++ b/tests/Elastic.Markdown.Tests/SiteMap/NavigationTestsBase.cs @@ -21,19 +21,17 @@ protected NavigationTestsBase(ITestOutputHelper output) { CurrentDirectory = Paths.Root.FullName }); - var context = new BuildContext + var context = new BuildContext(readFs, writeFs) { Force = false, UrlPathPrefix = null, - ReadFileSystem = readFs, - WriteFileSystem = writeFs, Collector = new DiagnosticsCollector(logger, []) }; var set = new DocumentationSet(context); set.Files.Should().HaveCountGreaterThan(10); - Generator = new DocumentationGenerator(set, context, logger); + Generator = new DocumentationGenerator(set, logger); }