Support cross-repository redirects and specialized anchor scenarios (#1227)

* Support cross-repository redirects and specialized anchor scenarios

* Removing leniency for asciidocalypse/unpublished links

Author: cotti

docs/contribute/redirects.md

@@ -49,6 +49,19 @@ redirects:
   'testing/redirects/third-page.md':
     anchors:
       'removed-anchor':
+  'testing/redirects/cross-repo-page.md': 'other-repo://reference/section/new-cross-repo-page.md'
+  'testing/redirects/8th-page.md':
+    to: 'other-repo://reference/section/new-cross-repo-page.md'
+    anchors: '!'
+    many:
+      - to: 'testing/redirects/second-page.md'
+        anchors:
+          'item-a': 'yy'
+      - to: 'testing/redirects/third-page.md'
+        anchors:
+          'item-b': 
+            
+  
 ```
 
 ### Redirect preserving all anchors
@@ -104,3 +117,49 @@ redirects:
       'old-anchor': 'active-anchor'
       'removed-anchor':
 ```
+
+### Redirecting to other repositories
+
+It is possible to redirect to other repositories. The syntax is the same as when linking on documentation sets:
+
+* 'other-repo://reference/section/new-cross-repo-page.md'
+
+```yaml
+redirects:
+  'testing/redirects/cross-repo-page.md': 'other-repo://reference/section/new-cross-repo-page.md'
+```
+
+### Managing complex scenarios with anchors
+
+* `to`, `anchor` and `many` can be used together to support more complex scenarios.
+* Setting `to` at the top level determines the default case, which can be used for partial redirects.
+* Cross-repository links are supported, with the same syntax as in the previous example.
+* The existing rules for `anchors` also apply here. To define a catch-all redirect, use `{}`.
+
+```yaml
+redirects:
+  # In this first scenario, the default redirection target remains the same page, with anchors being preserved. 
+  # Omitting the ``anchors`` tag or explicitly setting it as empty are both supported.
+  'testing/redirects/8th-page.md':
+    to: 'testing/redirects/8th-page.md'
+    many:
+      - to: 'testing/redirects/second-page.md'
+        anchors:
+          'item-a': 'yy'
+      - to: 'testing/redirects/third-page.md'
+        anchors:
+          'item-b':
+
+  # In this scenario, the default redirection target is a different page, and anchors are dropped.
+  'testing/redirects/deleted-page.md':
+    to: 'testing/redirects/5th-page.md'
+    anchors: '!'
+    many:
+      - to: "testing/redirects/second-page.md"
+        anchors:
+          "aa": "zz"
+          "removed-anchor":
+      - to: "other-repo://reference/section/partial-content.md"
+        anchors:
+          "bb": "yy"
+```

src/Elastic.Markdown/IO/DocumentationSet.cs

@@ -256,6 +256,14 @@ private void ValidateRedirectsExists()
 
 		void ValidateExists(string from, string to, IReadOnlyDictionary<string, string?>? valueAnchors)
 		{
+			if (to.Contains("://"))
+			{
+				if (!Uri.TryCreate(to, UriKind.Absolute, out _))
+					Context.EmitError(Configuration.SourceFile, $"Redirect {from} points to {to} which is not a valid URI");
+
+				return;
+			}
+
 			if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
 				to = to.Replace('/', Path.DirectorySeparatorChar);
 

src/Elastic.Markdown/Links/CrossLinks/CrossLinkResolver.cs

@@ -4,15 +4,14 @@
 
 using System.Collections.Frozen;
 using System.Diagnostics.CodeAnalysis;
-using Elastic.Documentation;
 using Elastic.Documentation.Links;
 
 namespace Elastic.Markdown.Links.CrossLinks;
 
 public interface ICrossLinkResolver
 {
 	Task<FetchedCrossLinks> FetchLinks(Cancel ctx);
-	bool TryResolve(Action<string> errorEmitter, Action<string> warningEmitter, Uri crossLinkUri, [NotNullWhen(true)] out Uri? resolvedUri);
+	bool TryResolve(Action<string> errorEmitter, Uri crossLinkUri, [NotNullWhen(true)] out Uri? resolvedUri);
 	IUriEnvironmentResolver UriResolver { get; }
 }
 
@@ -27,8 +26,8 @@ public async Task<FetchedCrossLinks> FetchLinks(Cancel ctx)
 		return _crossLinks;
 	}
 
-	public bool TryResolve(Action<string> errorEmitter, Action<string> warningEmitter, Uri crossLinkUri, [NotNullWhen(true)] out Uri? resolvedUri) =>
-		TryResolve(errorEmitter, warningEmitter, _crossLinks, UriResolver, crossLinkUri, out resolvedUri);
+	public bool TryResolve(Action<string> errorEmitter, Uri crossLinkUri, [NotNullWhen(true)] out Uri? resolvedUri) =>
+		TryResolve(errorEmitter, _crossLinks, UriResolver, crossLinkUri, out resolvedUri);
 
 	public FetchedCrossLinks UpdateLinkReference(string repository, RepositoryLinks repositoryLinks)
 	{
@@ -43,163 +42,161 @@ public FetchedCrossLinks UpdateLinkReference(string repository, RepositoryLinks
 
 	public static bool TryResolve(
 		Action<string> errorEmitter,
-		Action<string> warningEmitter,
 		FetchedCrossLinks fetchedCrossLinks,
 		IUriEnvironmentResolver uriResolver,
 		Uri crossLinkUri,
 		[NotNullWhen(true)] out Uri? resolvedUri
 	)
 	{
 		resolvedUri = null;
-		var lookup = fetchedCrossLinks.LinkReferences;
-		if (crossLinkUri.Scheme != "asciidocalypse" && lookup.TryGetValue(crossLinkUri.Scheme, out var linkReference))
-			return TryFullyValidate(errorEmitter, uriResolver, fetchedCrossLinks, linkReference, crossLinkUri, out resolvedUri);
 
-		// TODO this is temporary while we wait for all links.json to be published
-		// Here we just silently rewrite the cross_link to the url
-
-		var declaredRepositories = fetchedCrossLinks.DeclaredRepositories;
-		if (!declaredRepositories.Contains(crossLinkUri.Scheme))
+		if (!fetchedCrossLinks.LinkReferences.TryGetValue(crossLinkUri.Scheme, out var sourceLinkReference))
 		{
-			if (fetchedCrossLinks.FromConfiguration)
-				errorEmitter($"'{crossLinkUri.Scheme}' is not declared as valid cross link repository in docset.yml under cross_links: '{crossLinkUri}'");
-			else
-				warningEmitter($"'{crossLinkUri.Scheme}' is not yet publishing to the links registry: '{crossLinkUri}'");
+			errorEmitter($"'{crossLinkUri.Scheme}' was not found in the cross link index");
 			return false;
 		}
 
-		var lookupPath = (crossLinkUri.Host + '/' + crossLinkUri.AbsolutePath.TrimStart('/')).Trim('/');
-		var path = ToTargetUrlPath(lookupPath);
-		if (!string.IsNullOrEmpty(crossLinkUri.Fragment))
-			path += crossLinkUri.Fragment;
+		var originalLookupPath = (crossLinkUri.Host + '/' + crossLinkUri.AbsolutePath.TrimStart('/')).Trim('/');
+		if (string.IsNullOrEmpty(originalLookupPath) && crossLinkUri.Host.EndsWith(".md"))
+			originalLookupPath = crossLinkUri.Host;
 
-		resolvedUri = uriResolver.Resolve(crossLinkUri, path);
-		return true;
+		if (sourceLinkReference.Redirects is not null && sourceLinkReference.Redirects.TryGetValue(originalLookupPath, out var redirectRule))
+			return ResolveRedirect(errorEmitter, uriResolver, crossLinkUri, redirectRule, originalLookupPath, fetchedCrossLinks, out resolvedUri);
+
+		if (sourceLinkReference.Links.TryGetValue(originalLookupPath, out var directLinkMetadata))
+			return ResolveDirectLink(errorEmitter, uriResolver, crossLinkUri, originalLookupPath, directLinkMetadata, out resolvedUri);
+
+
+		var linksJson = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/elastic/{crossLinkUri.Scheme}/main/links.json";
+		if (fetchedCrossLinks.LinkIndexEntries.TryGetValue(crossLinkUri.Scheme, out var indexEntry))
+			linksJson = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/{indexEntry.Path}";
+
+		errorEmitter($"'{originalLookupPath}' is not a valid link in the '{crossLinkUri.Scheme}' cross link index: {linksJson}");
+		resolvedUri = null;
+		return false;
 	}
 
-	private static bool TryFullyValidate(Action<string> errorEmitter,
+	private static bool ResolveDirectLink(Action<string> errorEmitter,
 		IUriEnvironmentResolver uriResolver,
-		FetchedCrossLinks fetchedCrossLinks,
-		RepositoryLinks repositoryLinks,
 		Uri crossLinkUri,
+		string lookupPath,
+		LinkMetadata linkMetadata,
 		[NotNullWhen(true)] out Uri? resolvedUri)
 	{
 		resolvedUri = null;
-		var lookupPath = (crossLinkUri.Host + '/' + crossLinkUri.AbsolutePath.TrimStart('/')).Trim('/');
-		if (string.IsNullOrEmpty(lookupPath) && crossLinkUri.Host.EndsWith(".md"))
-			lookupPath = crossLinkUri.Host;
-
-		if (!LookupLink(errorEmitter, fetchedCrossLinks, repositoryLinks, crossLinkUri, ref lookupPath, out var link, out var lookupFragment))
-			return false;
-
-		var path = ToTargetUrlPath(lookupPath);
+		var lookupFragment = crossLinkUri.Fragment;
+		var targetUrlPath = ToTargetUrlPath(lookupPath);
 
 		if (!string.IsNullOrEmpty(lookupFragment))
 		{
-			if (link.Anchors is null)
-			{
-				errorEmitter($"'{lookupPath}' does not have any anchors so linking to '{crossLinkUri.Fragment}' is impossible.");
-				return false;
-			}
-
-			if (!link.Anchors.Contains(lookupFragment.TrimStart('#')))
+			var anchor = lookupFragment.TrimStart('#');
+			if (linkMetadata.Anchors is null || !linkMetadata.Anchors.Contains(anchor))
 			{
 				errorEmitter($"'{lookupPath}' has no anchor named: '{lookupFragment}'.");
 				return false;
 			}
 
-			path += "#" + lookupFragment.TrimStart('#');
+			targetUrlPath += lookupFragment;
 		}
 
-		resolvedUri = uriResolver.Resolve(crossLinkUri, path);
+		resolvedUri = uriResolver.Resolve(crossLinkUri, targetUrlPath);
 		return true;
 	}
 
-	private static bool LookupLink(Action<string> errorEmitter,
-		FetchedCrossLinks crossLinks,
-		RepositoryLinks repositoryLinks,
-		Uri crossLinkUri,
-		ref string lookupPath,
-		[NotNullWhen(true)] out LinkMetadata? link,
-		[NotNullWhen(true)] out string? lookupFragment)
+	private static bool ResolveRedirect(
+		Action<string> errorEmitter,
+		IUriEnvironmentResolver uriResolver,
+		Uri originalCrossLinkUri,
+		LinkRedirect redirectRule,
+		string originalLookupPath,
+		FetchedCrossLinks fetchedCrossLinks,
+		[NotNullWhen(true)] out Uri? resolvedUri)
 	{
-		lookupFragment = null;
+		resolvedUri = null;
+		var originalFragment = originalCrossLinkUri.Fragment.TrimStart('#');
 
-		if (repositoryLinks.Redirects is not null && repositoryLinks.Redirects.TryGetValue(lookupPath, out var redirect))
+		if (!string.IsNullOrEmpty(originalFragment) && redirectRule.Many is { Length: > 0 })
 		{
-			var targets = (redirect.Many ?? [])
-				.Select(r => r)
-				.Concat([redirect])
-				.Where(s => !string.IsNullOrEmpty(s.To))
-				.ToArray();
+			foreach (var subRule in redirectRule.Many)
+			{
+				if (string.IsNullOrEmpty(subRule.To))
+					continue;
 
-			return ResolveLinkRedirect(targets, errorEmitter, repositoryLinks, crossLinkUri, ref lookupPath, out link, ref lookupFragment);
+				if (subRule.Anchors is null || subRule.Anchors.Count == 0)
+					continue;
+
+				if (subRule.Anchors.TryGetValue("!", out _))
+					return FinalizeRedirect(errorEmitter, uriResolver, originalCrossLinkUri, subRule.To, null, fetchedCrossLinks, out resolvedUri);
+				if (subRule.Anchors.TryGetValue(originalFragment, out var mappedAnchor))
+					return FinalizeRedirect(errorEmitter, uriResolver, originalCrossLinkUri, subRule.To, mappedAnchor, fetchedCrossLinks, out resolvedUri);
+			}
 		}
 
-		if (repositoryLinks.Links.TryGetValue(lookupPath, out link))
+		string? finalTargetFragment = null;
+
+		if (!string.IsNullOrEmpty(originalFragment))
 		{
-			lookupFragment = crossLinkUri.Fragment;
-			return true;
+			if (redirectRule.Anchors?.TryGetValue("!", out _) ?? false)
+				finalTargetFragment = null;
+			else if (redirectRule.Anchors?.TryGetValue(originalFragment, out var mappedAnchor) ?? false)
+				finalTargetFragment = mappedAnchor;
+			else if (redirectRule.Anchors is null || redirectRule.Anchors.Count == 0)
+				finalTargetFragment = originalFragment;
+			else
+			{
+				errorEmitter($"Redirect rule for '{originalLookupPath}' in '{originalCrossLinkUri.Scheme}' found, but top-level rule did not handle anchor '#{originalFragment}'.");
+				return false;
+			}
 		}
 
-		var linksJson = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/elastic/{crossLinkUri.Scheme}/main/links.json";
-		if (crossLinks.LinkIndexEntries.TryGetValue(crossLinkUri.Scheme, out var linkIndexEntry))
-			linksJson = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/{linkIndexEntry.Path}";
-
-		errorEmitter($"'{lookupPath}' is not a valid link in the '{crossLinkUri.Scheme}' cross link index: {linksJson}");
-		return false;
+		return string.IsNullOrEmpty(redirectRule.To)
+			? FinalizeRedirect(errorEmitter, uriResolver, originalCrossLinkUri, originalLookupPath, finalTargetFragment, fetchedCrossLinks, out resolvedUri)
+			: FinalizeRedirect(errorEmitter, uriResolver, originalCrossLinkUri, redirectRule.To, finalTargetFragment, fetchedCrossLinks, out resolvedUri);
 	}
 
-	private static bool ResolveLinkRedirect(
-		LinkSingleRedirect[] redirects,
+	private static bool FinalizeRedirect(
 		Action<string> errorEmitter,
-		RepositoryLinks repositoryLinks,
-		Uri crossLinkUri,
-		ref string lookupPath, out LinkMetadata? link, ref string? lookupFragment)
+		IUriEnvironmentResolver uriResolver,
+		Uri originalProcessingUri,
+		string redirectToPath,
+		string? targetFragment,
+		FetchedCrossLinks fetchedCrossLinks,
+		[NotNullWhen(true)] out Uri? resolvedUri)
 	{
-		var fragment = crossLinkUri.Fragment.TrimStart('#');
-		link = null;
-		foreach (var redirect in redirects)
+		resolvedUri = null;
+		string finalPathForResolver;
+
+		if (Uri.TryCreate(redirectToPath, UriKind.Absolute, out var targetCrossUri) && targetCrossUri.Scheme != "http" && targetCrossUri.Scheme != "https")
 		{
-			if (string.IsNullOrEmpty(redirect.To))
-				continue;
-			if (!repositoryLinks.Links.TryGetValue(redirect.To, out link))
-				continue;
+			var lookupPath = $"{targetCrossUri.Host}/{targetCrossUri.AbsolutePath.TrimStart('/')}";
+			finalPathForResolver = ToTargetUrlPath(lookupPath);
 
-			if (string.IsNullOrEmpty(fragment))
-			{
-				lookupPath = redirect.To;
-				return true;
-			}
+			if (!string.IsNullOrEmpty(targetFragment) && targetFragment != "!")
+				finalPathForResolver += $"#{targetFragment}";
 
-			if (redirect.Anchors is null || redirect.Anchors.Count == 0)
+			if (!fetchedCrossLinks.LinkReferences.TryGetValue(targetCrossUri.Scheme, out var targetLinkReference))
 			{
-				if (redirects.Length > 1)
-					continue;
-				lookupPath = redirect.To;
-				lookupFragment = crossLinkUri.Fragment;
-				return true;
+				errorEmitter($"Redirect target '{redirectToPath}' points to repository '{targetCrossUri.Scheme}' for which no links.json was found.");
+				return false;
 			}
 
-			if (redirect.Anchors.TryGetValue("!", out _))
+			if (!targetLinkReference.Links.ContainsKey(lookupPath))
 			{
-				lookupPath = redirect.To;
-				lookupFragment = null;
-				return true;
+				errorEmitter($"Redirect target '{redirectToPath}' points to file '{lookupPath}' which was not found in repository '{targetCrossUri.Scheme}'s links.json.");
+				return false;
 			}
 
-			if (!redirect.Anchors.TryGetValue(crossLinkUri.Fragment.TrimStart('#'), out var newFragment))
-				continue;
-
-			lookupPath = redirect.To;
-			lookupFragment = newFragment;
-			return true;
+			resolvedUri = uriResolver.Resolve(targetCrossUri, finalPathForResolver); // Use targetUri for scheme and base
 		}
+		else
+		{
+			finalPathForResolver = ToTargetUrlPath(redirectToPath);
+			if (!string.IsNullOrEmpty(targetFragment) && targetFragment != "!")
+				finalPathForResolver += $"#{targetFragment}";
 
-		var targets = string.Join(", ", redirects.Select(r => r.To));
-		var failedLookup = lookupFragment is null ? lookupPath : $"{lookupPath}#{lookupFragment.TrimStart('#')}";
-		errorEmitter($"'{failedLookup}' is set a redirect but none of redirect '{targets}' match or exist in links.json.");
-		return false;
+			resolvedUri = uriResolver.Resolve(originalProcessingUri, finalPathForResolver); // Use original URI's scheme
+		}
+		return true;
 	}
 
 	private static string ToTargetUrlPath(string lookupPath)

src/Elastic.Markdown/Links/InboundLinks/LinkIndexLinkChecker.cs

@@ -126,7 +126,7 @@ RepositoryFilter filter
 					}
 
 					collector.EmitError(repository, s);
-				}, s => collector.EmitWarning(linksJson, s), uri, out _);
+				}, uri, out _);
 			}
 		}
 		// non-strict for now

src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs

@@ -184,7 +184,6 @@ private static void ProcessCrossLink(LinkInline link, InlineProcessor processor,
 
 		if (context.CrossLinkResolver.TryResolve(
 				s => processor.EmitError(link, s),
-				s => processor.EmitWarning(link, s),
 				uri, out var resolvedUri)
 			 )
 			link.Url = resolvedUri.ToString();