Merge branch 'main' into ecs-9.0

Mpdreamz · web-flow · commit 96f33ab515f2 · 2025-05-21T20:09:34.000+02:00
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -14,7 +14,7 @@ env:
   NUGET_PACKAGES: ${{ github.workspace }}/.nuget/packages
 
 jobs:
-  validate-navigation:
+  validate-assembler:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -23,9 +23,12 @@ jobs:
         id: bootstrap
         uses: ./.github/actions/bootstrap
 
-      - name: Build
+      - name: Validate Navigation
         run: dotnet run --project src/tooling/docs-assembler -c release -- navigation validate
 
+      - name: Validate Content Sources
+        run: dotnet run --project src/tooling/docs-assembler -c release -- content-source validate
+
   build-lambda:
     uses: ./.github/workflows/build-link-index-updater-lambda.yml
 
diff --git a/.github/workflows/preview-build.yml b/.github/workflows/preview-build.yml
@@ -61,6 +61,7 @@ jobs:
       content-source-match: ${{ steps.event-check.outputs.content-source-match != '' && steps.event-check.outputs.content-source-match || steps.match.outputs.content-source-match }}
       content-source-next: ${{ steps.event-check.outputs.content-source-next != '' && steps.event-check.outputs.content-source-next || steps.match.outputs.content-source-next }}
       content-source-current: ${{ steps.event-check.outputs.content-source-current != '' && steps.event-check.outputs.content-source-current || steps.match.outputs.content-source-current }}
+      content-source-speculative: ${{ steps.event-check.outputs.content-source-speculative != '' && steps.event-check.outputs.content-source-speculative || steps.match.outputs.content-source-speculative }}
     steps:
       - name: Not a push event
         id: event-check
@@ -70,6 +71,7 @@ jobs:
           echo "content-source-match=true" >> $GITHUB_OUTPUT
           echo "content-source-next=false" >> $GITHUB_OUTPUT
           echo "content-source-current=false" >> $GITHUB_OUTPUT
+          echo "content-source-speculative=false" >> $GITHUB_OUTPUT
       - name: Match for push events
         id: match
         if: contains(fromJSON('["push"]'), github.event_name)
@@ -81,8 +83,9 @@ jobs:
         run: |
           echo "Non sensitive data, echo'ing here temporarily to validate this job before connecting it further into the build job"
           echo "content-source-match=${{ steps.event-check.outputs.content-source-match != '' && steps.event-check.outputs.content-source-match || steps.match.outputs.content-source-match }}"
-          echo "content-source-name=${{ steps.event-check.outputs.content-source-next != '' && steps.event-check.outputs.content-source-next || steps.match.outputs.content-source-next }}"
+          echo "content-source-next=${{ steps.event-check.outputs.content-source-next != '' && steps.event-check.outputs.content-source-next || steps.match.outputs.content-source-next }}"
           echo "content-source-current=${{ steps.event-check.outputs.content-source-current != '' && steps.event-check.outputs.content-source-current || steps.match.outputs.content-source-current }}"
+          echo "content-source-speculative=${{ steps.event-check.outputs.content-source-speculative != '' && steps.event-check.outputs.content-source-speculative || steps.match.outputs.content-source-speculative }}"
           echo "ref=${{ github.ref_name }}"
           echo "repo=${{ github.repository }}"
 
@@ -233,7 +236,11 @@ jobs:
         if: |
           env.MATCH == 'true' && 
           (contains(fromJSON('["push", "workflow_dispatch"]'), github.event_name) 
-            && (needs.match.outputs.content-source-current == 'true' || needs.match.outputs.content-source-next == 'true') 
+            && (
+                needs.match.outputs.content-source-current == 'true' 
+                || needs.match.outputs.content-source-next == 'true'
+                || needs.match.outputs.content-source-speculative == 'true'
+            ) 
             && steps.s3-upload.outcome == 'success')
         uses: elastic/docs-builder/actions/update-link-index@main
 
diff --git a/actions/assembler-config-validate/action.yml b/actions/assembler-config-validate/action.yml
@@ -0,0 +1,12 @@
+name: 'Documentation Assembler Configuration Validator'
+description: 'Ensures the assembler configuration is valid'
+
+branding:
+  icon: 'filter'
+  color: 'blue'
+
+runs:
+  using: 'docker'
+  image: "docker://ghcr.io/elastic/docs-assembler:edge"
+  env:
+    INPUT_COMMAND: "content-source validate"
diff --git a/actions/assembler-match/action.yml b/actions/assembler-match/action.yml
@@ -20,6 +20,8 @@ outputs:
     description: "true/false indicating the branch acts as the next content source"
   content-source-current:
     description: "true/false indicating the branch acts as the current content source"
+  content-source-speculative:
+    description: "true/false speculative match, used to build version branches before they are marked as current/next"
 
 runs:
   using: 'docker'
diff --git a/docs-builder.sln b/docs-builder.sln
@@ -94,6 +94,11 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "assembler-match", "assemble
 		actions\assembler-match\action.yml = actions\assembler-match\action.yml
 	EndProjectSection
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "assembler-config-validate", "assembler-config-validate", "{E20FEEF9-1D1A-4CDA-A546-7FDC573BE399}"
+	ProjectSection(SolutionItems) = preProject
+		actions\assembler-config-validate\action.yml = actions\assembler-config-validate\action.yml
+	EndProjectSection
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -178,5 +183,6 @@ Global
 		{059E787F-85C1-43BE-9DD6-CE319E106383} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
 		{7D36DDDA-9E0B-4D2C-8033-5D62FF8B6166} = {059E787F-85C1-43BE-9DD6-CE319E106383}
 		{FB1C1954-D8E2-4745-BA62-04DD82FB4792} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
+		{E20FEEF9-1D1A-4CDA-A546-7FDC573BE399} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
 	EndGlobalSection
 EndGlobal
diff --git a/docs/contribute/locally.md b/docs/contribute/locally.md
@@ -44,11 +44,12 @@ This guide uses the first option. If you'd like to clone the repository and buil
 
 2. **Run docs-builder from a docs folder**
 
-   Use the `serve` command from any docs folder to start serving the documentation at http://localhost:3000. The path to the `docset.yml` file that you want to build can be specified with `-p`:
+   Use the `serve` command from any docs folder to start serving the documentation at http://localhost:3000:
 
    ```sh
    docs-builder serve
    ```
+   The path to the `docset.yml` file that you want to build can be specified with `-p`.
 
 To download and install the binary file manually, refer to [Releases](https://github.com/elastic/docs-builder/releases) on GitHub. 
 
@@ -72,12 +73,12 @@ If you get a `Permission denied` error, make sure that you aren't trying to run
 
 2. **Run docs-builder from a docs folder**
 
-   Use the `serve` command from any docs folder to start serving the documentation at http://localhost:3000. The path to the `docset.yml` file that you want to build can be specified with `-p`:
+   Use the `serve` command from any docs folder to start serving the documentation at http://localhost:3000:
 
    ```sh
    docs-builder serve
    ```
-   
+   The path to the `docset.yml` file that you want to build can be specified with `-p`.
 :::
 ::::
 
@@ -111,7 +112,7 @@ cd docs-content
 
 :::::{step} Run docs-builder
 
-Run the `docs-builder` binary with the `serve` command to build and serve the content set to http://localhost:3000. Specify the path to the `docset.yml` file that you want to build with `-p`.
+Run the `docs-builder` binary with the `serve` command to build and serve the content set to http://localhost:3000. If necessary, specify the path to the `docset.yml` file that you want to build with `-p`.
 
 For example:
 
diff --git a/src/Elastic.Documentation.Configuration/Assembler/AssemblyConfiguration.cs b/src/Elastic.Documentation.Configuration/Assembler/AssemblyConfiguration.cs
@@ -2,6 +2,7 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using System.Text.RegularExpressions;
 using Elastic.Documentation.Serialization;
 using YamlDotNet.Serialization;
 using YamlStaticContext = Elastic.Documentation.Configuration.Serialization.YamlStaticContext;
@@ -83,19 +84,45 @@ private static TRepository RepositoryDefaults<TRepository>(TRepository r, string
 	/// <paramref name="repository"/>.
 	public ContentSourceMatch Match(string repository, string branchOrTag)
 	{
-		var repositoryName = repository.Split('/').Last();
-		var match = new ContentSourceMatch(null, null);
+		var match = new ContentSourceMatch(null, null, false);
+		var tokens = repository.Split('/');
+		var repositoryName = tokens.Last();
+		var owner = tokens.First();
+
+		if (tokens.Length < 2 || owner != "elastic")
+			return match;
+
 		if (ReferenceRepositories.TryGetValue(repositoryName, out var r))
 		{
-			if (r.GetBranch(ContentSource.Current) == branchOrTag)
+			var current = r.GetBranch(ContentSource.Current);
+			var next = r.GetBranch(ContentSource.Next);
+			var isVersionBranch = ContentSourceRegex.MatchVersionBranch().IsMatch(branchOrTag);
+			if (current == branchOrTag)
 				match = match with { Current = ContentSource.Current };
-			if (r.GetBranch(ContentSource.Next) == branchOrTag)
+			if (next == branchOrTag)
 				match = match with { Next = ContentSource.Next };
+			if (isVersionBranch && SemVersion.TryParse(branchOrTag + ".0", out var v))
+			{
+				// if the current branch is a version, only speculatively match if branch is actually a new version
+				if (SemVersion.TryParse(current + ".0", out var currentVersion))
+				{
+					if (v >= currentVersion)
+						match = match with { Speculative = true };
+				}
+				// assume we are newly onboarding the repository to current/next
+				else
+					match = match with { Speculative = true };
+			}
 			return match;
 		}
 
 		if (repositoryName != NarrativeRepository.RepositoryName)
-			return match;
+		{
+			// this is an unknown new elastic repository
+			var isVersionBranch = ContentSourceRegex.MatchVersionBranch().IsMatch(branchOrTag);
+			if (isVersionBranch || branchOrTag == "main" || branchOrTag == "master")
+				return match with { Speculative = true };
+		}
 
 		if (Narrative.GetBranch(ContentSource.Current) == branchOrTag)
 			match = match with { Current = ContentSource.Current };
@@ -105,5 +132,12 @@ public ContentSourceMatch Match(string repository, string branchOrTag)
 		return match;
 	}
 
-	public record ContentSourceMatch(ContentSource? Current, ContentSource? Next);
+	public record ContentSourceMatch(ContentSource? Current, ContentSource? Next, bool Speculative);
+
+}
+
+internal static partial class ContentSourceRegex
+{
+	[GeneratedRegex(@"^\d+\.\d+$", RegexOptions.IgnoreCase, "en-US")]
+	public static partial Regex MatchVersionBranch();
 }
diff --git a/src/Elastic.Documentation/SemVersion.cs b/src/Elastic.Documentation/SemVersion.cs
@@ -6,7 +6,7 @@
 using System.Globalization;
 using System.Text.RegularExpressions;
 
-namespace Elastic.Markdown.Helpers;
+namespace Elastic.Documentation;
 
 /// <summary>
 /// A semver2 compatible version.
diff --git a/src/Elastic.Markdown/Links/CrossLinks/CrossLinkFetcher.cs b/src/Elastic.Markdown/Links/CrossLinks/CrossLinkFetcher.cs
@@ -33,6 +33,7 @@ public record FetchedCrossLinks
 
 public abstract class CrossLinkFetcher(ILoggerFactory logger) : IDisposable
 {
+	public const string RegistryUrl = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/link-index.json";
 	private readonly ILogger _logger = logger.CreateLogger(nameof(CrossLinkFetcher));
 	private readonly HttpClient _client = new();
 	private LinkReferenceRegistry? _linkIndex;
@@ -42,16 +43,16 @@ public static LinkReference Deserialize(string json) =>
 
 	public abstract Task<FetchedCrossLinks> Fetch(Cancel ctx);
 
-	protected async Task<LinkReferenceRegistry> FetchLinkIndex(Cancel ctx)
+	public async Task<LinkReferenceRegistry> FetchLinkIndex(Cancel ctx)
 	{
 		if (_linkIndex is not null)
 		{
 			_logger.LogTrace("Using cached link index");
 			return _linkIndex;
 		}
-		var url = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/link-index.json";
-		_logger.LogInformation("Fetching {Url}", url);
-		var json = await _client.GetStringAsync(url, ctx);
+
+		_logger.LogInformation("Fetching {Url}", RegistryUrl);
+		var json = await _client.GetStringAsync(RegistryUrl, ctx);
 		_linkIndex = LinkReferenceRegistry.Deserialize(json);
 		return _linkIndex;
 	}
diff --git a/src/Elastic.Markdown/Myst/Directives/VersionBlock.cs b/src/Elastic.Markdown/Myst/Directives/VersionBlock.cs
@@ -2,6 +2,7 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using Elastic.Documentation;
 using Elastic.Markdown.Diagnostics;
 using Elastic.Markdown.Helpers;
 using static System.StringSplitOptions;
diff --git a/src/Elastic.Markdown/Myst/FrontMatter/AllVersions.cs b/src/Elastic.Markdown/Myst/FrontMatter/AllVersions.cs
@@ -2,6 +2,7 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using Elastic.Documentation;
 using Elastic.Markdown.Helpers;
 using YamlDotNet.Core;
 using YamlDotNet.Core.Events;
diff --git a/src/Elastic.Markdown/Myst/FrontMatter/Applicability.cs b/src/Elastic.Markdown/Myst/FrontMatter/Applicability.cs
@@ -5,6 +5,7 @@
 using System.Collections;
 using System.Diagnostics.CodeAnalysis;
 using System.Text;
+using Elastic.Documentation;
 using Elastic.Markdown.Helpers;
 using YamlDotNet.Serialization;
 
diff --git a/src/Elastic.Markdown/Myst/Roles/AppliesTo/AppliesToRole.cs b/src/Elastic.Markdown/Myst/Roles/AppliesTo/AppliesToRole.cs
@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information
 
 using System.Diagnostics;
+using Elastic.Documentation;
 using Elastic.Markdown.Diagnostics;
 using Elastic.Markdown.Helpers;
 using Elastic.Markdown.Myst.FrontMatter;
diff --git a/src/tooling/docs-assembler/Building/AssemblerCrossLinkFetcher.cs b/src/tooling/docs-assembler/Building/AssemblerCrossLinkFetcher.cs
@@ -11,7 +11,8 @@
 
 namespace Documentation.Assembler.Building;
 
-public class AssemblerCrossLinkFetcher(ILoggerFactory logger, AssemblyConfiguration configuration, PublishEnvironment publishEnvironment) : CrossLinkFetcher(logger)
+public class AssemblerCrossLinkFetcher(ILoggerFactory logger, AssemblyConfiguration configuration, PublishEnvironment publishEnvironment)
+	: CrossLinkFetcher(logger)
 {
 	public override async Task<FetchedCrossLinks> Fetch(Cancel ctx)
 	{
diff --git a/src/tooling/docs-assembler/Cli/ContentSourceCommands.cs b/src/tooling/docs-assembler/Cli/ContentSourceCommands.cs
@@ -6,8 +6,10 @@
 using System.IO.Abstractions;
 using Actions.Core.Services;
 using ConsoleAppFramework;
+using Documentation.Assembler.Building;
 using Elastic.Documentation.Configuration.Assembler;
 using Elastic.Documentation.Tooling.Diagnostics.Console;
+using Elastic.Markdown.Links.CrossLinks;
 using Microsoft.Extensions.Logging;
 
 namespace Documentation.Assembler.Cli;
@@ -22,6 +24,54 @@ private void AssignOutputLogger()
 		ConsoleApp.LogError = msg => log.LogError(msg);
 	}
 
+	[Command("validate")]
+	public async Task<int> Validate(Cancel ctx = default)
+	{
+		AssignOutputLogger();
+		await using var collector = new ConsoleDiagnosticsCollector(logFactory, githubActionsService)
+		{
+			NoHints = true
+		};
+
+		_ = collector.StartAsync(ctx);
+
+		// environment does not matter to check the configuration, defaulting to dev
+		var context = new AssembleContext("dev", collector, new FileSystem(), new FileSystem(), null, null)
+		{
+			Force = false,
+			AllowIndexing = false
+		};
+		var fetcher = new AssemblerCrossLinkFetcher(logFactory, context.Configuration, context.Environment);
+		var links = await fetcher.FetchLinkIndex(ctx);
+		var repositories = context.Configuration.ReferenceRepositories.Values.Concat<Repository>([context.Configuration.Narrative]).ToList();
+
+		foreach (var repository in repositories)
+		{
+			if (!links.Repositories.TryGetValue(repository.Name, out var registryMapping))
+			{
+				collector.EmitError(context.ConfigurationPath, $"'{repository}' does not exist in {CrossLinkFetcher.RegistryUrl}");
+				continue;
+			}
+
+			var current = repository.GetBranch(ContentSource.Current);
+			var next = repository.GetBranch(ContentSource.Next);
+			if (!registryMapping.TryGetValue(next, out _))
+			{
+				collector.EmitError(context.ConfigurationPath,
+					$"'{repository.Name}' has not yet published links.json for configured 'next' content source: '{next}' see  {CrossLinkFetcher.RegistryUrl}");
+			}
+			if (!registryMapping.TryGetValue(current, out _))
+			{
+				collector.EmitError(context.ConfigurationPath,
+					$"'{repository.Name}' has not yet published links.json for configured 'current' content source: '{current}' see  {CrossLinkFetcher.RegistryUrl}");
+			}
+		}
+
+
+		await collector.StopAsync(ctx);
+		return collector.Errors == 0 ? 0 : 1;
+	}
+
 	/// <summary>  </summary>
 	/// <param name="repository"></param>
 	/// <param name="branchOrTag"></param>
@@ -55,12 +105,13 @@ public async Task<int> Match([Argument] string? repository = null, [Argument] st
 			AllowIndexing = false
 		};
 		var matches = assembleContext.Configuration.Match(repo, refName);
-		if (matches is { Current: null, Next: null })
+		if (matches is { Current: null, Next: null, Speculative: false })
 		{
 			logger.LogInformation("'{Repository}' '{BranchOrTag}' combination not found in configuration.", repo, refName);
 			await githubActionsService.SetOutputAsync("content-source-match", "false");
 			await githubActionsService.SetOutputAsync("content-source-next", "false");
 			await githubActionsService.SetOutputAsync("content-source-current", "false");
+			await githubActionsService.SetOutputAsync("content-source-speculative", "false");
 		}
 		else
 		{
@@ -72,6 +123,7 @@ public async Task<int> Match([Argument] string? repository = null, [Argument] st
 			await githubActionsService.SetOutputAsync("content-source-match", "true");
 			await githubActionsService.SetOutputAsync("content-source-next", matches.Next is not null ? "true" : "false");
 			await githubActionsService.SetOutputAsync("content-source-current", matches.Current is not null ? "true" : "false");
+			await githubActionsService.SetOutputAsync("content-source-speculative", matches.Speculative ? "true" : "false");
 		}
 
 		await collector.StopAsync(ctx);
diff --git a/src/tooling/docs-builder/Cli/CheckForUpdatesFilter.cs b/src/tooling/docs-builder/Cli/CheckForUpdatesFilter.cs
@@ -4,6 +4,7 @@
 
 using System.Reflection;
 using ConsoleAppFramework;
+using Elastic.Documentation;
 using Elastic.Markdown.Helpers;
 using Elastic.Markdown.IO;
 
diff --git a/tests/Elastic.Markdown.Tests/Directives/VersionTests.cs b/tests/Elastic.Markdown.Tests/Directives/VersionTests.cs
@@ -2,6 +2,7 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using Elastic.Documentation;
 using Elastic.Markdown.Helpers;
 using Elastic.Markdown.Myst.Directives;
 using FluentAssertions;

Original file line number	Diff line number	Diff line change
`@@ -33,6 +33,7 @@ public record FetchedCrossLinks`
`33`	`33`
`34`	`34`	`public abstract class CrossLinkFetcher(ILoggerFactory logger) : IDisposable`
`35`	`35`	`{`
	`36`	`+ public const string RegistryUrl = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/link-index.json";`
`36`	`37`	`private readonly ILogger _logger = logger.CreateLogger(nameof(CrossLinkFetcher));`
`37`	`38`	`private readonly HttpClient _client = new();`
`38`	`39`	`private LinkReferenceRegistry? _linkIndex;`
`@@ -42,16 +43,16 @@ public static LinkReference Deserialize(string json) =>`
`42`	`43`
`43`	`44`	`public abstract Task<FetchedCrossLinks> Fetch(Cancel ctx);`
`44`	`45`
`45`		`- protected async Task<LinkReferenceRegistry> FetchLinkIndex(Cancel ctx)`
	`46`	`+ public async Task<LinkReferenceRegistry> FetchLinkIndex(Cancel ctx)`
`46`	`47`	`{`
`47`	`48`	`if (_linkIndex is not null)`
`48`	`49`	`{`
`49`	`50`	`_logger.LogTrace("Using cached link index");`
`50`	`51`	`return _linkIndex;`
`51`	`52`	`}`
`52`		`- var url = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/link-index.json";`
`53`		`- _logger.LogInformation("Fetching {Url}", url);`
`54`		`- var json = await _client.GetStringAsync(url, ctx);`
	`53`	`+`
	`54`	`+ _logger.LogInformation("Fetching {Url}", RegistryUrl);`
	`55`	`+ var json = await _client.GetStringAsync(RegistryUrl, ctx);`
`55`	`56`	`_linkIndex = LinkReferenceRegistry.Deserialize(json);`
`56`	`57`	`return _linkIndex;`
`57`	`58`	`}`