Simplify building nodes without intermediary nav items

Author: Mpdreamz

Directory.Packages.props

@@ -18,14 +18,15 @@
     <PackageVersion Include="Amazon.Lambda.S3Events" Version="3.1.0" />
     <PackageVersion Include="Amazon.Lambda.Serialization.SystemTextJson" Version="2.4.4" />
     <PackageVersion Include="Amazon.Lambda.SQSEvents" Version="2.2.0" />
+    <PackageVersion Include="AngleSharp" Version="1.1.2" />
     <PackageVersion Include="Aspire.Hosting" Version="9.4.2" />
     <PackageVersion Include="Aspire.Hosting.Testing" Version="9.4.2" />
     <PackageVersion Include="AWSSDK.Core" Version="4.0.0.2" />
     <PackageVersion Include="AWSSDK.SQS" Version="4.0.0.1" />
     <PackageVersion Include="AWSSDK.S3" Version="4.0.0.1" />
     <PackageVersion Include="Elastic.OpenTelemetry" Version="1.1.0" />
     <PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.12.0" />
-    <PackageVersion Include="Generator.Equals" Version="3.2.1"  PrivateAssets="all" IncludeAssets="runtime; build; native; contentfiles; analyzers; buildtransitive"/>
+    <PackageVersion Include="Generator.Equals" Version="3.2.1" PrivateAssets="all" IncludeAssets="runtime; build; native; contentfiles; analyzers; buildtransitive" />
     <PackageVersion Include="KubernetesClient" Version="17.0.14" />
     <PackageVersion Include="Elastic.Aspire.Hosting.Elasticsearch" Version="9.3.0" />
     <PackageVersion Include="Elastic.Clients.Elasticsearch" Version="9.1.4" />

config/versions.yml

@@ -20,6 +20,7 @@ versioning_systems:
   eck:
     base: 3.0
     current: 3.1.0
+    
   ess: *all
   ecs:
     base: 9.0

src/Elastic.ApiExplorer/Landing/LandingNavigationItem.cs

@@ -36,6 +36,7 @@ public class LandingNavigationItem : IApiGroupingNavigationItem<ApiLanding, INav
 	public bool IsCrossLink => false; // API landing items are never cross-links
 	public string Url => Index.Url;
 	public bool Hidden => false;
+	public Uri Identifier { get; } = new Uri("todo://");
 
 	public string NavigationTitle => Index.NavigationTitle;
 
@@ -50,6 +51,9 @@ public LandingNavigationItem(string url)
 
 	/// <inheritdoc />
 	public bool IsUsingNavigationDropdown => false;
+
+	void IAssignableChildrenNavigation.SetNavigationItems(IReadOnlyCollection<INavigationItem> navigationItems) =>
+		throw new NotSupportedException($"{nameof(IAssignableChildrenNavigation.SetNavigationItems)} is not supported on ${nameof(ClassificationNavigationItem)}");
 }
 
 public interface IApiGroupingNavigationItem<out TGroupingModel, out TNavigationItem> : INodeNavigationItem<TGroupingModel, TNavigationItem>
@@ -64,6 +68,7 @@ INodeNavigationItem<INavigationModel, INavigationItem> parent
 	: IApiGroupingNavigationItem<TGroupingModel, TNavigationItem>
 	where TGroupingModel : IApiGroupingModel
 	where TNavigationItem : INavigationItem
+
 {
 	/// <inheritdoc />
 	public string Url => NavigationItems.First().Url;
@@ -83,6 +88,8 @@ INodeNavigationItem<INavigationModel, INavigationItem> parent
 	public int NavigationIndex { get; set; }
 	public bool IsCrossLink => false; // API grouping items are never cross-links
 
+	public Uri Identifier { get; } = new Uri("todo://");
+
 	/// <inheritdoc />
 	public int Depth => 0;
 
@@ -108,6 +115,9 @@ public class ClassificationNavigationItem(ApiClassification classification, Land
 
 	/// <inheritdoc />
 	public bool IsUsingNavigationDropdown => false;
+
+	void IAssignableChildrenNavigation.SetNavigationItems(IReadOnlyCollection<INavigationItem> navigationItems) =>
+		throw new NotSupportedException($"{nameof(IAssignableChildrenNavigation.SetNavigationItems)} is not supported on ${nameof(ClassificationNavigationItem)}");
 }
 
 public class TagNavigationItem(ApiTag tag, IRootNavigationItem<IApiGroupingModel, INavigationItem> rootNavigation, INodeNavigationItem<INavigationModel, INavigationItem> parent)

src/Elastic.Documentation.Configuration/DocSet/SiteNavigationFile.cs

@@ -52,6 +52,24 @@ public static bool ValidatePathPrefixes(IDiagnosticsCollector collector, SiteNav
 		return valid;
 	}
 
+	public static ImmutableHashSet<Uri> GetAllDeclaredSources(SiteNavigationFile siteNavigation)
+	{
+		var set = new HashSet<Uri>();
+
+		foreach (var tocRef in siteNavigation.TableOfContents)
+			CollectSource(tocRef, set);
+
+		return set.ToImmutableHashSet();
+	}
+	private static void CollectSource(SiteTableOfContentsRef tocRef, HashSet<Uri> set)
+	{
+		_ = set.Add(tocRef.Source);
+		// Recursively collect from children
+		foreach (var child in tocRef.Children)
+			CollectSource(child, set);
+	}
+
+
 	public static ImmutableHashSet<Uri> GetAllPathPrefixes(SiteNavigationFile siteNavigation)
 	{
 		var set = new HashSet<Uri>();

src/Elastic.Documentation.Links/CrossLinks/CrossLinkFetcher.cs

@@ -39,11 +39,14 @@ public static RepositoryLinks Deserialize(string json) =>
 
 	public abstract Task<FetchedCrossLinks> FetchCrossLinks(Cancel ctx);
 
+	private int _logOnce;
 	public async Task<LinkRegistry> FetchLinkRegistry(Cancel ctx)
 	{
+		var result = Interlocked.Increment(ref _logOnce);
 		if (_linkIndex is not null)
 		{
-			Logger.LogTrace("Using cached link index registry (link-index.json)");
+			if (result == 1)
+				Logger.LogTrace("Using cached link index registry (link-index.json)");
 			return _linkIndex;
 		}
 

src/Elastic.Documentation.Navigation/Assembler/AssembledNavigation.cs

@@ -2,6 +2,7 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using System.Collections.Immutable;
 using System.Diagnostics;
 using Elastic.Documentation.Configuration.Assembler;
 using Elastic.Documentation.Configuration.DocSet;
@@ -18,7 +19,8 @@ public class SiteNavigation : IRootNavigationItem<IDocumentationFile, INavigatio
 {
 	private readonly string? _sitePrefix;
 
-	public SiteNavigation(SiteNavigationFile siteNavigationFile,
+	public SiteNavigation(
+		SiteNavigationFile siteNavigationFile,
 		IDocumentationContext context,
 		IReadOnlyCollection<IDocumentationSetNavigation> documentationSetNavigations,
 		string? sitePrefix
@@ -35,6 +37,9 @@ public SiteNavigation(SiteNavigationFile siteNavigationFile,
 		Id = ShortId.Create("site");
 		IsUsingNavigationDropdown = false;
 		Phantoms = siteNavigationFile.Phantoms;
+		DeclaredPhantoms = [.. siteNavigationFile.Phantoms.Select(p => new Uri(p.Source))];
+		DeclaredTableOfContents = SiteNavigationFile.GetAllDeclaredSources(siteNavigationFile);
+
 		_nodes = [];
 		foreach (var setNavigation in documentationSetNavigations)
 		{
@@ -73,13 +78,21 @@ public SiteNavigation(SiteNavigationFile siteNavigationFile,
 		Index = this.FindIndex<IDocumentationFile>(new NotFoundModel("/index.md"));
 	}
 
-	private readonly Dictionary<Uri, INodeNavigationItem<IDocumentationFile, INavigationItem>> _nodes;
-	public IReadOnlyDictionary<Uri, INodeNavigationItem<IDocumentationFile, INavigationItem>> Nodes => _nodes;
+	public HashSet<Uri> DeclaredPhantoms { get; }
+
+	/// <summary> All the table of contents explicitly declared in the navigation</summary>
+	public ImmutableHashSet<Uri> DeclaredTableOfContents { get; set; }
+
+	private readonly Dictionary<Uri, IRootNavigationItem<IDocumentationFile, INavigationItem>> _nodes;
+	public IReadOnlyDictionary<Uri, IRootNavigationItem<IDocumentationFile, INavigationItem>> Nodes => _nodes;
 
 	private HashSet<Uri> UnseenNodes { get; }
 
 	public IReadOnlyCollection<PhantomRegistration> Phantoms { get; }
 
+	/// <inheritdoc />
+	public Uri Identifier { get; } = new Uri("site://");
+
 	//TODO Obsolete?
 	public IReadOnlyCollection<INodeNavigationItem<INavigationModel, INavigationItem>> TopLevelItems =>
 		NavigationItems.OfType<INodeNavigationItem<INavigationModel, INavigationItem>>().ToList();
@@ -120,6 +133,9 @@ public SiteNavigation(SiteNavigationFile siteNavigationFile,
 	/// <inheritdoc />
 	public IReadOnlyCollection<INavigationItem> NavigationItems { get; }
 
+	void IAssignableChildrenNavigation.SetNavigationItems(IReadOnlyCollection<INavigationItem> navigationItems) =>
+		throw new NotSupportedException("SetNavigationItems is not supported on SiteNavigation");
+
 	/// <summary>
 	/// Normalizes the site prefix to ensure it has a leading slash and no trailing slash.
 	/// Returns null for null or empty/whitespace input.
@@ -156,20 +172,20 @@ public SiteNavigation(SiteNavigationFile siteNavigationFile,
 			// we allow not setting path prefixes for toc references from the narrative repository
 			if (tocRef.Source.Scheme != NarrativeRepository.RepositoryName)
 				context.EmitError(context.ConfigurationPath, $"path_prefix is required for TOC reference: {tocRef.Source}");
-
-			if (!string.IsNullOrEmpty(tocRef.Source.Host))
-				pathPrefix += $"/{tocRef.Source.Host}";
-			if (!string.IsNullOrEmpty(tocRef.Source.AbsolutePath) && tocRef.Source.AbsolutePath != "/")
-				pathPrefix += $"/{tocRef.Source.AbsolutePath}";
+			else
+			{
+				if (!string.IsNullOrEmpty(tocRef.Source.Host))
+					pathPrefix += $"/{tocRef.Source.Host}";
+				if (!string.IsNullOrEmpty(tocRef.Source.AbsolutePath) && tocRef.Source.AbsolutePath != "/")
+					pathPrefix += $"/{tocRef.Source.AbsolutePath}";
+			}
 		}
 
 		// Normalize pathPrefix to remove leading/trailing slashes for a consistent combination
 		pathPrefix = pathPrefix.Trim('/');
 
 		// Combine with site prefix if present, otherwise ensure leading slash
-		pathPrefix = !string.IsNullOrWhiteSpace(_sitePrefix)
-			? $"{_sitePrefix}/{pathPrefix}"
-			: "/" + pathPrefix;
+		pathPrefix = !string.IsNullOrWhiteSpace(_sitePrefix) ? $"{_sitePrefix}/{pathPrefix}" : "/" + pathPrefix;
 
 		// Look up the node in the collected nodes
 		if (!_nodes.TryGetValue(tocRef.Source, out var node))
@@ -183,25 +199,33 @@ public SiteNavigation(SiteNavigationFile siteNavigationFile,
 			return null;
 		}
 
+		if (node.Identifier == new Uri("docs-content://reference/glossary"))
+		{
+		}
+
+		root ??= node;
+
 		_ = UnseenNodes.Remove(tocRef.Source);
 		// Set the navigation index
+		node.Parent = parent;
 		node.NavigationIndex = index;
-		homeAccessor.HomeProvider = new NavigationHomeProvider(pathPrefix, root ?? homeAccessor.HomeProvider.NavigationRoot);
+		homeAccessor.HomeProvider = new NavigationHomeProvider(pathPrefix, root);
 
-		var wrapped = new SiteTableOfContentsNavigation<IDocumentationFile>(node, homeAccessor.HomeProvider, parent, root);
-		parent = wrapped;
-		root ??= wrapped.NavigationRoot;
+		//var wrapped = new SiteTableOfContentsNavigation<IDocumentationFile>(node, homeAccessor.HomeProvider, parent, root);
+		//parent = wrapped;
 
 		var children = new List<INavigationItem>();
 		var nodeChildren = node.NavigationItems;
 		foreach (var nodeChild in nodeChildren)
 		{
-			nodeChild.Parent = parent;
-			if (nodeChild is IRootNavigationItem<IDocumentationFile, INavigationItem>)
+			nodeChild.Parent = node;
+			if (nodeChild is INavigationHomeAccessor childAccessor)
+				childAccessor.HomeProvider = homeAccessor.HomeProvider;
+
+			if (nodeChild is IRootNavigationItem<INavigationModel, INavigationItem>)
 				continue;
 			children.Add(nodeChild);
 		}
-		// Recursively create child navigation items if children are specified
 		if (tocRef.Children.Count > 0)
 		{
 			var childIndex = 0;
@@ -218,10 +242,27 @@ public SiteNavigation(SiteNavigationFile siteNavigationFile,
 					children.Add(childItem);
 			}
 		}
+		foreach (var nodeChild in nodeChildren)
+		{
+			if (nodeChild is not IRootNavigationItem<INavigationModel, INavigationItem> rootChild)
+				continue;
+			if (DeclaredTableOfContents.Contains(rootChild.Identifier) || DeclaredPhantoms.Contains(rootChild.Identifier))
+				continue;
+
+			context.EmitWarning(context.ConfigurationPath, $"Navigation does not explicitly declare: {rootChild.Identifier}");
+		}
 
-		wrapped.NavigationItems = children;
-		// Always return a wrapper to ensure path_prefix is the URL (not path_prefix + node's URL)
-		return wrapped;
+		switch (node)
+		{
+			case SiteNavigation:
+				break;
+			case IAssignableChildrenNavigation documentationSetNavigation:
+				documentationSetNavigation.SetNavigationItems(children);
+				break;
+			default:
+				throw new Exception($"node is not a known type: {node.GetType().Name}");
+		}
+		return node;
 	}
 }
 

src/Elastic.Documentation.Navigation/INavigationItem.cs

@@ -71,9 +71,16 @@ public interface INodeNavigationItem<out TIndex, out TChildNavigation> : INaviga
 	IReadOnlyCollection<TChildNavigation> NavigationItems { get; }
 }
 
-public interface IRootNavigationItem<out TIndex, out TChildNavigation> : INodeNavigationItem<TIndex, TChildNavigation>
+public interface IAssignableChildrenNavigation
+{
+	void SetNavigationItems(IReadOnlyCollection<INavigationItem> navigationItems);
+}
+
+public interface IRootNavigationItem<out TIndex, out TChildNavigation> : INodeNavigationItem<TIndex, TChildNavigation>, IAssignableChildrenNavigation
 	where TIndex : INavigationModel
 	where TChildNavigation : INavigationItem
 {
 	bool IsUsingNavigationDropdown { get; }
+
+	Uri Identifier { get; }
 }

src/Elastic.Documentation.Navigation/Isolated/CrossLinkNavigationLeaf.cs

@@ -14,12 +14,10 @@ public class CrossLinkNavigationLeaf(
 	string url,
 	bool hidden,
 	INodeNavigationItem<INavigationModel, INavigationItem>? parent,
-	INavigationHomeProvider homeProvider
+	INavigationHomeAccessor homeAccessor
 )
 	: ILeafNavigationItem<CrossLinkModel>
 {
-	private readonly INavigationHomeProvider _homeProvider = homeProvider;
-
 	/// <inheritdoc />
 	public CrossLinkModel Model { get; init; } = model;
 
@@ -30,7 +28,7 @@ INavigationHomeProvider homeProvider
 	public bool Hidden { get; init; } = hidden;
 
 	/// <inheritdoc />
-	public IRootNavigationItem<INavigationModel, INavigationItem> NavigationRoot => _homeProvider.NavigationRoot;
+	public IRootNavigationItem<INavigationModel, INavigationItem> NavigationRoot => homeAccessor.HomeProvider.NavigationRoot;
 
 	/// <inheritdoc />
 	public INodeNavigationItem<INavigationModel, INavigationItem>? Parent { get; set; } = parent;