From d6320f1e278f6a7b69558bce6807fc82179be9db Mon Sep 17 00:00:00 2001 From: piergm <134913285+piergm@users.noreply.github.com> Date: Thu, 28 Aug 2025 14:38:55 +0200 Subject: [PATCH 1/6] WIP --- .../action/AuthorizedProjectsSupplier.java | 33 ++++++ .../action/CPSExpressionRewriter.java | 100 ++++++++++++++++++ .../elasticsearch/action/IndicesRequest.java | 37 +++++++ .../transport/RemoteClusterAware.java | 2 +- 4 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 server/src/main/java/org/elasticsearch/action/AuthorizedProjectsSupplier.java create mode 100644 server/src/main/java/org/elasticsearch/action/CPSExpressionRewriter.java diff --git a/server/src/main/java/org/elasticsearch/action/AuthorizedProjectsSupplier.java b/server/src/main/java/org/elasticsearch/action/AuthorizedProjectsSupplier.java new file mode 100644 index 0000000000000..9c7b07cddc7d5 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/AuthorizedProjectsSupplier.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action; + +import org.elasticsearch.core.Nullable; + +import java.util.List; + +public interface AuthorizedProjectsSupplier { + AuthorizedProjects get(); + + class Default implements AuthorizedProjectsSupplier { + @Override + public AuthorizedProjects get() { + return AuthorizedProjects.NOT_CROSS_PROJECT; + } + } + + record AuthorizedProjects(@Nullable String origin, List projects) { + public static AuthorizedProjects NOT_CROSS_PROJECT = new AuthorizedProjects(null, List.of()); + + public boolean isOriginOnly() { + return origin != null && projects.isEmpty(); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/action/CPSExpressionRewriter.java b/server/src/main/java/org/elasticsearch/action/CPSExpressionRewriter.java new file mode 100644 index 0000000000000..470a47c477d94 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/action/CPSExpressionRewriter.java @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.logging.LogManager; +import org.elasticsearch.logging.Logger; +import org.elasticsearch.transport.RemoteClusterAware; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.elasticsearch.transport.RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; +import static org.elasticsearch.transport.RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR; + +public class CPSExpressionRewriter { + private static final Logger logger = LogManager.getLogger(CPSExpressionRewriter.class); + + public static void maybeRewriteCrossProjectResolvableRequest( + RemoteClusterAware remoteClusterAware, + AuthorizedProjectsSupplier.AuthorizedProjects targetProjects, + IndicesRequest.CrossProjectReplaceable request + ) throws ResourceNotFoundException { + if (targetProjects == AuthorizedProjectsSupplier.AuthorizedProjects.NOT_CROSS_PROJECT) { + logger.info("Cross-project search is disabled or not applicable, skipping request [{}]...", request); + return; + } + + if (targetProjects.isOriginOnly()) { + logger.info("Cross-project search is only for the origin project [{}], skipping rewrite...", targetProjects.origin()); + return; + } + + if (targetProjects.projects().isEmpty()) { + throw new ResourceNotFoundException("no target projects for cross-project search request"); + } + + List projects = targetProjects.projects(); + String[] indices = request.indices(); + logger.info("Rewriting indices for CPS [{}]", Arrays.toString(indices)); + + Map canonicalExpressionsMap = new LinkedHashMap<>(indices.length); + for (String indexExpression : indices) { + // TODO we need to handle exclusions here already + boolean isQualified = isQualifiedIndexExpression(indexExpression); + if (isQualified) { + // TODO handle empty case here -- empty means "search all" in ES which is _not_ what we want + List canonicalExpressions = rewriteQualified(indexExpression, projects, remoteClusterAware); + // could fail early here in ignore_unavailable and allow_no_indices strict mode if things are empty + canonicalExpressionsMap.put(indexExpression, new IndicesRequest.CanonicalExpression(indexExpression, canonicalExpressions)); + logger.info("Rewrote qualified expression [{}] to [{}]", indexExpression, canonicalExpressions); + } else { + // un-qualified expression, i.e. flat-world + List canonicalExpressions = rewriteUnqualified(indexExpression, targetProjects.projects()); + canonicalExpressionsMap.put(indexExpression, new IndicesRequest.CanonicalExpression(indexExpression, canonicalExpressions)); + logger.info("Rewrote unqualified expression [{}] to [{}]", indexExpression, canonicalExpressions); + } + } + + request.setCanonicalExpressions(canonicalExpressionsMap); + } + + private static List rewriteUnqualified(String indexExpression, List projects) { + List canonicalExpressions = new ArrayList<>(); + canonicalExpressions.add(indexExpression); + for (String targetProject : projects) { + canonicalExpressions.add(RemoteClusterAware.buildRemoteIndexName(targetProject, indexExpression)); + } + return canonicalExpressions; + } + + private static List rewriteQualified(String indicesExpressions, List projects, RemoteClusterAware remoteClusterAware) { + final Map> map = remoteClusterAware.groupClusterIndices( + Set.copyOf(projects), + new String[] { indicesExpressions } + ); + final List local = map.remove(LOCAL_CLUSTER_GROUP_KEY); + final List remote = map.entrySet() + .stream() + .flatMap(e -> e.getValue().stream().map(v -> e.getKey() + REMOTE_CLUSTER_INDEX_SEPARATOR + v)) + .toList(); + assert local == null || local.isEmpty() : "local indices should not be present in the map, but were: " + local; + return remote; + } + + public static boolean isQualifiedIndexExpression(String indexExpression) { + return RemoteClusterAware.isRemoteIndexName(indexExpression); + } +} diff --git a/server/src/main/java/org/elasticsearch/action/IndicesRequest.java b/server/src/main/java/org/elasticsearch/action/IndicesRequest.java index 176ecbc7a2395..b183d774f13bb 100644 --- a/server/src/main/java/org/elasticsearch/action/IndicesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/IndicesRequest.java @@ -10,9 +10,13 @@ package org.elasticsearch.action; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.shard.ShardId; import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; /** * Needs to be implemented by all {@link org.elasticsearch.action.ActionRequest} subclasses that relate to @@ -62,6 +66,37 @@ interface Replaceable extends IndicesRequest { default boolean allowsRemoteIndices() { return false; } + + default void setCanonicalExpressions(@Nullable Map canonicalExpressions) { + if (false == storeCanonicalExpressions()) { + assert false : "setCanonicalExpressions should not be called when storeCanonicalExpressions is false"; + throw new IllegalStateException("setCanonicalExpressions should not be called when storeCanonicalExpressions is false"); + } + } + + default Map getCanonicalExpressions() { + if (false == storeCanonicalExpressions()) { + assert false : "getCanonicalExpressions should not be called when storeCanonicalExpressions is false"; + throw new IllegalStateException("getCanonicalExpressions should not be called when storeCanonicalExpressions is false"); + } + return new LinkedHashMap<>(); + } + + default boolean storeCanonicalExpressions() { + return false; + } + } + + interface CrossProjectReplaceable extends Replaceable { + @Override + default boolean allowsRemoteIndices() { + return true; + } + + @Override + default boolean storeCanonicalExpressions() { + return true; + } } /** @@ -91,4 +126,6 @@ interface RemoteClusterShardRequest extends IndicesRequest { */ Collection shards(); } + + record CanonicalExpression(String originalExpression, List canonicalExpressions) {} } diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java index 23e4049c264f8..8f687c568fe9a 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterAware.java @@ -135,7 +135,7 @@ public static String[] splitIndexName(String indexExpression) { * * @return a map of grouped remote and local indices */ - protected Map> groupClusterIndices(Set remoteClusterNames, String[] requestIndices) { + public Map> groupClusterIndices(Set remoteClusterNames, String[] requestIndices) { Map> perClusterIndices = new HashMap<>(); Set clustersToRemove = new HashSet<>(); for (String index : requestIndices) { From 4cb1f32ce6dd9f36670f4f7ec6a8a0bc220ad55d Mon Sep 17 00:00:00 2001 From: piergm <134913285+piergm@users.noreply.github.com> Date: Fri, 29 Aug 2025 13:52:25 +0200 Subject: [PATCH 2/6] some simplification --- .../org/elasticsearch/action/CPSExpressionRewriter.java | 6 +++--- .../main/java/org/elasticsearch/action/IndicesRequest.java | 6 ++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/CPSExpressionRewriter.java b/server/src/main/java/org/elasticsearch/action/CPSExpressionRewriter.java index 470a47c477d94..3a7b06e43a903 100644 --- a/server/src/main/java/org/elasticsearch/action/CPSExpressionRewriter.java +++ b/server/src/main/java/org/elasticsearch/action/CPSExpressionRewriter.java @@ -50,7 +50,7 @@ public static void maybeRewriteCrossProjectResolvableRequest( String[] indices = request.indices(); logger.info("Rewriting indices for CPS [{}]", Arrays.toString(indices)); - Map canonicalExpressionsMap = new LinkedHashMap<>(indices.length); + Map> canonicalExpressionsMap = new LinkedHashMap<>(indices.length); for (String indexExpression : indices) { // TODO we need to handle exclusions here already boolean isQualified = isQualifiedIndexExpression(indexExpression); @@ -58,12 +58,12 @@ public static void maybeRewriteCrossProjectResolvableRequest( // TODO handle empty case here -- empty means "search all" in ES which is _not_ what we want List canonicalExpressions = rewriteQualified(indexExpression, projects, remoteClusterAware); // could fail early here in ignore_unavailable and allow_no_indices strict mode if things are empty - canonicalExpressionsMap.put(indexExpression, new IndicesRequest.CanonicalExpression(indexExpression, canonicalExpressions)); + canonicalExpressionsMap.put(indexExpression, canonicalExpressions); logger.info("Rewrote qualified expression [{}] to [{}]", indexExpression, canonicalExpressions); } else { // un-qualified expression, i.e. flat-world List canonicalExpressions = rewriteUnqualified(indexExpression, targetProjects.projects()); - canonicalExpressionsMap.put(indexExpression, new IndicesRequest.CanonicalExpression(indexExpression, canonicalExpressions)); + canonicalExpressionsMap.put(indexExpression, canonicalExpressions); logger.info("Rewrote unqualified expression [{}] to [{}]", indexExpression, canonicalExpressions); } } diff --git a/server/src/main/java/org/elasticsearch/action/IndicesRequest.java b/server/src/main/java/org/elasticsearch/action/IndicesRequest.java index b183d774f13bb..e843f9e9540b3 100644 --- a/server/src/main/java/org/elasticsearch/action/IndicesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/IndicesRequest.java @@ -67,14 +67,14 @@ default boolean allowsRemoteIndices() { return false; } - default void setCanonicalExpressions(@Nullable Map canonicalExpressions) { + default void setCanonicalExpressions(@Nullable Map> canonicalExpressions) { if (false == storeCanonicalExpressions()) { assert false : "setCanonicalExpressions should not be called when storeCanonicalExpressions is false"; throw new IllegalStateException("setCanonicalExpressions should not be called when storeCanonicalExpressions is false"); } } - default Map getCanonicalExpressions() { + default Map> getCanonicalExpressions() { if (false == storeCanonicalExpressions()) { assert false : "getCanonicalExpressions should not be called when storeCanonicalExpressions is false"; throw new IllegalStateException("getCanonicalExpressions should not be called when storeCanonicalExpressions is false"); @@ -126,6 +126,4 @@ interface RemoteClusterShardRequest extends IndicesRequest { */ Collection shards(); } - - record CanonicalExpression(String originalExpression, List canonicalExpressions) {} } From 2d1903894e2bfa818ba8bdadd2f2d15528f89b5e Mon Sep 17 00:00:00 2001 From: piergm <134913285+piergm@users.noreply.github.com> Date: Fri, 29 Aug 2025 14:47:52 +0200 Subject: [PATCH 3/6] some more tests + handing of _origin --- .../action/CPSExpressionRewriter.java | 25 ++- .../action/CPSExpressionRewriterTests.java | 205 ++++++++++++++++++ 2 files changed, 226 insertions(+), 4 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/action/CPSExpressionRewriterTests.java diff --git a/server/src/main/java/org/elasticsearch/action/CPSExpressionRewriter.java b/server/src/main/java/org/elasticsearch/action/CPSExpressionRewriter.java index 3a7b06e43a903..86aa0766e5544 100644 --- a/server/src/main/java/org/elasticsearch/action/CPSExpressionRewriter.java +++ b/server/src/main/java/org/elasticsearch/action/CPSExpressionRewriter.java @@ -26,6 +26,7 @@ public class CPSExpressionRewriter { private static final Logger logger = LogManager.getLogger(CPSExpressionRewriter.class); + private static final String WILDCARD = "*"; public static void maybeRewriteCrossProjectResolvableRequest( RemoteClusterAware remoteClusterAware, @@ -46,7 +47,6 @@ public static void maybeRewriteCrossProjectResolvableRequest( throw new ResourceNotFoundException("no target projects for cross-project search request"); } - List projects = targetProjects.projects(); String[] indices = request.indices(); logger.info("Rewriting indices for CPS [{}]", Arrays.toString(indices)); @@ -56,7 +56,7 @@ public static void maybeRewriteCrossProjectResolvableRequest( boolean isQualified = isQualifiedIndexExpression(indexExpression); if (isQualified) { // TODO handle empty case here -- empty means "search all" in ES which is _not_ what we want - List canonicalExpressions = rewriteQualified(indexExpression, projects, remoteClusterAware); + List canonicalExpressions = rewriteQualified(indexExpression, targetProjects, remoteClusterAware); // could fail early here in ignore_unavailable and allow_no_indices strict mode if things are empty canonicalExpressionsMap.put(indexExpression, canonicalExpressions); logger.info("Rewrote qualified expression [{}] to [{}]", indexExpression, canonicalExpressions); @@ -80,9 +80,18 @@ private static List rewriteUnqualified(String indexExpression, List rewriteQualified(String indicesExpressions, List projects, RemoteClusterAware remoteClusterAware) { + private static List rewriteQualified( + String indicesExpressions, + AuthorizedProjectsSupplier.AuthorizedProjects targetProjects, + RemoteClusterAware remoteClusterAware + ) { + String[] splitExpression = RemoteClusterAware.splitIndexName(indicesExpressions); + if (targetProjects.origin() != null && targetProjects.origin().equals(splitExpression[0])) { + // handling special case where we have a qualified expression like: _origin:indexName + return List.of(splitExpression[1]); + } final Map> map = remoteClusterAware.groupClusterIndices( - Set.copyOf(projects), + Set.copyOf(targetProjects.projects()), new String[] { indicesExpressions } ); final List local = map.remove(LOCAL_CLUSTER_GROUP_KEY); @@ -91,6 +100,14 @@ private static List rewriteQualified(String indicesExpressions, List e.getValue().stream().map(v -> e.getKey() + REMOTE_CLUSTER_INDEX_SEPARATOR + v)) .toList(); assert local == null || local.isEmpty() : "local indices should not be present in the map, but were: " + local; + if (WILDCARD.equals(splitExpression[0])) { + // handing of special case where the original expression was: *:indexName that is a + // qualified expression that includes the origin cluster and all linked projects. + List remoteIncludingOrigin = new ArrayList<>(remote.size() + 1); + remoteIncludingOrigin.addAll(remote); + remoteIncludingOrigin.add(splitExpression[1]); + return remoteIncludingOrigin; + } return remote; } diff --git a/server/src/test/java/org/elasticsearch/action/CPSExpressionRewriterTests.java b/server/src/test/java/org/elasticsearch/action/CPSExpressionRewriterTests.java new file mode 100644 index 0000000000000..296369f3b1697 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/CPSExpressionRewriterTests.java @@ -0,0 +1,205 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.action; + +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.transport.RemoteClusterAware; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.hamcrest.Matchers.containsInAnyOrder; + +public class CPSExpressionRewriterTests extends ESTestCase { + private RemoteClusterAwareTest remoteClusterAware = new RemoteClusterAwareTest(); + + public void testFlatOnlyRewriteCrossProjectResolvableRequest() { + AuthorizedProjectsSupplier.AuthorizedProjects authorizedProjects = new AuthorizedProjectsSupplier.AuthorizedProjects( + "_origin", + List.of("P1", "P2", "P3") + ); + CrossProjectReplaceableTest crossProjectRequest = new CrossProjectReplaceableTest( + new String[] { "logs*", "metrics*" }, + IndicesOptions.DEFAULT + ); + + CPSExpressionRewriter.maybeRewriteCrossProjectResolvableRequest(remoteClusterAware, authorizedProjects, crossProjectRequest); + + assertThat(crossProjectRequest.getCanonicalExpressions().keySet(), containsInAnyOrder("logs*", "metrics*")); + assertThat( + crossProjectRequest.getCanonicalExpressions().get("logs*"), + containsInAnyOrder("logs*", "P1:logs*", "P2:logs*", "P3:logs*") + ); + assertThat( + crossProjectRequest.getCanonicalExpressions().get("metrics*"), + containsInAnyOrder("metrics*", "P1:metrics*", "P2:metrics*", "P3:metrics*") + ); + } + + public void testQualifiedOnlyLinkedProjectsRewrite() { + AuthorizedProjectsSupplier.AuthorizedProjects authorizedProjects = new AuthorizedProjectsSupplier.AuthorizedProjects( + "_origin", + List.of("P1", "P2", "P3") + ); + CrossProjectReplaceableTest crossProjectRequest = new CrossProjectReplaceableTest( + new String[] { "P1:logs*", "P2:metrics*" }, + IndicesOptions.DEFAULT + ); + + CPSExpressionRewriter.maybeRewriteCrossProjectResolvableRequest(remoteClusterAware, authorizedProjects, crossProjectRequest); + + assertThat(crossProjectRequest.getCanonicalExpressions().keySet(), containsInAnyOrder("P1:logs*", "P2:metrics*")); + assertThat(crossProjectRequest.getCanonicalExpressions().get("P1:logs*"), containsInAnyOrder("P1:logs*")); + assertThat(crossProjectRequest.getCanonicalExpressions().get("P2:metrics*"), containsInAnyOrder("P2:metrics*")); + } + + public void testQualifiedOnlyOriginProjectsRewrite() { + AuthorizedProjectsSupplier.AuthorizedProjects authorizedProjects = new AuthorizedProjectsSupplier.AuthorizedProjects( + "_origin", + List.of("P1", "P2", "P3") + ); + + CrossProjectReplaceableTest crossProjectRequest = new CrossProjectReplaceableTest( + new String[] { "_origin:logs*", "_origin:metrics*" }, + IndicesOptions.DEFAULT + ); + + CPSExpressionRewriter.maybeRewriteCrossProjectResolvableRequest(remoteClusterAware, authorizedProjects, crossProjectRequest); + + assertThat(crossProjectRequest.getCanonicalExpressions().keySet(), containsInAnyOrder("_origin:logs*", "_origin:metrics*")); + assertThat(crossProjectRequest.getCanonicalExpressions().get("_origin:logs*"), containsInAnyOrder("logs*")); + assertThat(crossProjectRequest.getCanonicalExpressions().get("_origin:metrics*"), containsInAnyOrder("metrics*")); + } + + public void testQualifiedOriginAndLikedProjectsRewrite() { + AuthorizedProjectsSupplier.AuthorizedProjects authorizedProjects = new AuthorizedProjectsSupplier.AuthorizedProjects( + "_origin", + List.of("P1", "P2", "P3") + ); + CrossProjectReplaceableTest crossProjectRequest = new CrossProjectReplaceableTest( + new String[] { "P1:logs*", "_origin:metrics*" }, + IndicesOptions.DEFAULT + ); + + CPSExpressionRewriter.maybeRewriteCrossProjectResolvableRequest(remoteClusterAware, authorizedProjects, crossProjectRequest); + + assertThat(crossProjectRequest.getCanonicalExpressions().keySet(), containsInAnyOrder("P1:logs*", "_origin:metrics*")); + assertThat(crossProjectRequest.getCanonicalExpressions().get("P1:logs*"), containsInAnyOrder("P1:logs*")); + assertThat(crossProjectRequest.getCanonicalExpressions().get("_origin:metrics*"), containsInAnyOrder("metrics*")); + } + + public void testQualifiedStartsWithProjectWildcardRewrite() { + AuthorizedProjectsSupplier.AuthorizedProjects authorizedProjects = new AuthorizedProjectsSupplier.AuthorizedProjects( + "_origin", + List.of("P1", "P2", "Q1", "Q2") + ); + CrossProjectReplaceableTest crossProjectRequest = new CrossProjectReplaceableTest( + new String[] { "P*:metrics*" }, + IndicesOptions.DEFAULT + ); + + CPSExpressionRewriter.maybeRewriteCrossProjectResolvableRequest(remoteClusterAware, authorizedProjects, crossProjectRequest); + + assertThat(crossProjectRequest.getCanonicalExpressions().keySet(), containsInAnyOrder("P*:metrics*")); + assertThat(crossProjectRequest.getCanonicalExpressions().get("P*:metrics*"), containsInAnyOrder("P1:metrics*", "P2:metrics*")); + } + + public void testQualifiedEndsWithProjectWildcardRewrite() { + AuthorizedProjectsSupplier.AuthorizedProjects authorizedProjects = new AuthorizedProjectsSupplier.AuthorizedProjects( + "_origin", + List.of("P1", "P2", "Q1", "Q2") + ); + CrossProjectReplaceableTest crossProjectRequest = new CrossProjectReplaceableTest( + new String[] { "*1:metrics*" }, + IndicesOptions.DEFAULT + ); + + CPSExpressionRewriter.maybeRewriteCrossProjectResolvableRequest(remoteClusterAware, authorizedProjects, crossProjectRequest); + + assertThat(crossProjectRequest.getCanonicalExpressions().keySet(), containsInAnyOrder("*1:metrics*")); + assertThat(crossProjectRequest.getCanonicalExpressions().get("*1:metrics*"), containsInAnyOrder("P1:metrics*", "Q1:metrics*")); + } + + public void testWildcardOnlyProjectRewrite() { + AuthorizedProjectsSupplier.AuthorizedProjects authorizedProjects = new AuthorizedProjectsSupplier.AuthorizedProjects( + "_origin", + List.of("P1", "P2", "Q1", "Q2") + ); + CrossProjectReplaceableTest crossProjectRequest = new CrossProjectReplaceableTest( + new String[] { "*:metrics*" }, + IndicesOptions.DEFAULT + ); + + CPSExpressionRewriter.maybeRewriteCrossProjectResolvableRequest(remoteClusterAware, authorizedProjects, crossProjectRequest); + + assertThat(crossProjectRequest.getCanonicalExpressions().keySet(), containsInAnyOrder("*:metrics*")); + assertThat( + crossProjectRequest.getCanonicalExpressions().get("*:metrics*"), + containsInAnyOrder("P1:metrics*", "P2:metrics*", "Q1:metrics*", "Q2:metrics*", "metrics*") + ); + } + + private static class RemoteClusterAwareTest extends RemoteClusterAware { + RemoteClusterAwareTest() { + super(Settings.EMPTY); + } + + @Override + protected void updateRemoteCluster(String clusterAlias, Settings settings) { + + } + + @Override + public Map> groupClusterIndices(Set remoteClusterNames, String[] requestIndices) { + return super.groupClusterIndices(remoteClusterNames, requestIndices); + } + } + + private static class CrossProjectReplaceableTest implements IndicesRequest.CrossProjectReplaceable { + private String[] indices; + private IndicesOptions options; + private Map> canonicalExpressions; + + CrossProjectReplaceableTest(String[] indices, IndicesOptions options) { + this.indices = indices; + this.options = options; + } + + @Override + public IndicesRequest indices(String... indices) { + this.indices = indices; + return this; + } + + @Override + public String[] indices() { + return indices; + } + + @Override + public IndicesOptions indicesOptions() { + return options; + } + + @Override + public void setCanonicalExpressions(Map> canonicalExpressions) { + this.canonicalExpressions = canonicalExpressions; + } + + @Override + public Map> getCanonicalExpressions() { + // Maybe this should be a record that contains also if the expression was flat or not. + return canonicalExpressions; + } + } +} From e101fc28707dfb56ff722204bbc7aa9130a16949 Mon Sep 17 00:00:00 2001 From: piergm <134913285+piergm@users.noreply.github.com> Date: Fri, 29 Aug 2025 15:48:23 +0200 Subject: [PATCH 4/6] iter --- .../action/CPSExpressionRewriterTests.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/action/CPSExpressionRewriterTests.java b/server/src/test/java/org/elasticsearch/action/CPSExpressionRewriterTests.java index 296369f3b1697..b438a7ded2eec 100644 --- a/server/src/test/java/org/elasticsearch/action/CPSExpressionRewriterTests.java +++ b/server/src/test/java/org/elasticsearch/action/CPSExpressionRewriterTests.java @@ -46,6 +46,26 @@ public void testFlatOnlyRewriteCrossProjectResolvableRequest() { ); } + public void testFlatAndQualifiedProjectRewrite() { + AuthorizedProjectsSupplier.AuthorizedProjects authorizedProjects = new AuthorizedProjectsSupplier.AuthorizedProjects( + "_origin", + List.of("P1", "P2", "P3") + ); + CrossProjectReplaceableTest crossProjectRequest = new CrossProjectReplaceableTest( + new String[] { "P1:logs*", "metrics*" }, + IndicesOptions.DEFAULT + ); + + CPSExpressionRewriter.maybeRewriteCrossProjectResolvableRequest(remoteClusterAware, authorizedProjects, crossProjectRequest); + + assertThat(crossProjectRequest.getCanonicalExpressions().keySet(), containsInAnyOrder("P1:logs*", "metrics*")); + assertThat(crossProjectRequest.getCanonicalExpressions().get("P1:logs*"), containsInAnyOrder("P1:logs*")); + assertThat( + crossProjectRequest.getCanonicalExpressions().get("metrics*"), + containsInAnyOrder("metrics*", "P1:metrics*", "P2:metrics*", "P3:metrics*") + ); + } + public void testQualifiedOnlyLinkedProjectsRewrite() { AuthorizedProjectsSupplier.AuthorizedProjects authorizedProjects = new AuthorizedProjectsSupplier.AuthorizedProjects( "_origin", From 6ea3219f3314bd13a73441dd8ae4170baf5e4a84 Mon Sep 17 00:00:00 2001 From: piergm <134913285+piergm@users.noreply.github.com> Date: Fri, 29 Aug 2025 16:31:56 +0200 Subject: [PATCH 5/6] handle match all --- ...er.java => CrossProjectResolverUtils.java} | 9 ++- ...va => CrossProjectResolverUtilsTests.java} | 58 ++++++++++++++++--- 2 files changed, 56 insertions(+), 11 deletions(-) rename server/src/main/java/org/elasticsearch/action/{CPSExpressionRewriter.java => CrossProjectResolverUtils.java} (93%) rename server/src/test/java/org/elasticsearch/action/{CPSExpressionRewriterTests.java => CrossProjectResolverUtilsTests.java} (72%) diff --git a/server/src/main/java/org/elasticsearch/action/CPSExpressionRewriter.java b/server/src/main/java/org/elasticsearch/action/CrossProjectResolverUtils.java similarity index 93% rename from server/src/main/java/org/elasticsearch/action/CPSExpressionRewriter.java rename to server/src/main/java/org/elasticsearch/action/CrossProjectResolverUtils.java index 86aa0766e5544..85fd313626c38 100644 --- a/server/src/main/java/org/elasticsearch/action/CPSExpressionRewriter.java +++ b/server/src/main/java/org/elasticsearch/action/CrossProjectResolverUtils.java @@ -24,9 +24,10 @@ import static org.elasticsearch.transport.RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; import static org.elasticsearch.transport.RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR; -public class CPSExpressionRewriter { - private static final Logger logger = LogManager.getLogger(CPSExpressionRewriter.class); +public class CrossProjectResolverUtils { + private static final Logger logger = LogManager.getLogger(CrossProjectResolverUtils.class); private static final String WILDCARD = "*"; + private static final String MATCH_ALL = "_ALL"; public static void maybeRewriteCrossProjectResolvableRequest( RemoteClusterAware remoteClusterAware, @@ -50,6 +51,10 @@ public static void maybeRewriteCrossProjectResolvableRequest( String[] indices = request.indices(); logger.info("Rewriting indices for CPS [{}]", Arrays.toString(indices)); + if (indices.length == 0 || WILDCARD.equals(indices[0]) || MATCH_ALL.equalsIgnoreCase(indices[0])) { + // handling of match all cases + indices = new String[] { WILDCARD }; + } Map> canonicalExpressionsMap = new LinkedHashMap<>(indices.length); for (String indexExpression : indices) { // TODO we need to handle exclusions here already diff --git a/server/src/test/java/org/elasticsearch/action/CPSExpressionRewriterTests.java b/server/src/test/java/org/elasticsearch/action/CrossProjectResolverUtilsTests.java similarity index 72% rename from server/src/test/java/org/elasticsearch/action/CPSExpressionRewriterTests.java rename to server/src/test/java/org/elasticsearch/action/CrossProjectResolverUtilsTests.java index b438a7ded2eec..8d1e24d79a63e 100644 --- a/server/src/test/java/org/elasticsearch/action/CPSExpressionRewriterTests.java +++ b/server/src/test/java/org/elasticsearch/action/CrossProjectResolverUtilsTests.java @@ -20,7 +20,7 @@ import static org.hamcrest.Matchers.containsInAnyOrder; -public class CPSExpressionRewriterTests extends ESTestCase { +public class CrossProjectResolverUtilsTests extends ESTestCase { private RemoteClusterAwareTest remoteClusterAware = new RemoteClusterAwareTest(); public void testFlatOnlyRewriteCrossProjectResolvableRequest() { @@ -33,7 +33,7 @@ public void testFlatOnlyRewriteCrossProjectResolvableRequest() { IndicesOptions.DEFAULT ); - CPSExpressionRewriter.maybeRewriteCrossProjectResolvableRequest(remoteClusterAware, authorizedProjects, crossProjectRequest); + CrossProjectResolverUtils.maybeRewriteCrossProjectResolvableRequest(remoteClusterAware, authorizedProjects, crossProjectRequest); assertThat(crossProjectRequest.getCanonicalExpressions().keySet(), containsInAnyOrder("logs*", "metrics*")); assertThat( @@ -56,7 +56,7 @@ public void testFlatAndQualifiedProjectRewrite() { IndicesOptions.DEFAULT ); - CPSExpressionRewriter.maybeRewriteCrossProjectResolvableRequest(remoteClusterAware, authorizedProjects, crossProjectRequest); + CrossProjectResolverUtils.maybeRewriteCrossProjectResolvableRequest(remoteClusterAware, authorizedProjects, crossProjectRequest); assertThat(crossProjectRequest.getCanonicalExpressions().keySet(), containsInAnyOrder("P1:logs*", "metrics*")); assertThat(crossProjectRequest.getCanonicalExpressions().get("P1:logs*"), containsInAnyOrder("P1:logs*")); @@ -76,7 +76,7 @@ public void testQualifiedOnlyLinkedProjectsRewrite() { IndicesOptions.DEFAULT ); - CPSExpressionRewriter.maybeRewriteCrossProjectResolvableRequest(remoteClusterAware, authorizedProjects, crossProjectRequest); + CrossProjectResolverUtils.maybeRewriteCrossProjectResolvableRequest(remoteClusterAware, authorizedProjects, crossProjectRequest); assertThat(crossProjectRequest.getCanonicalExpressions().keySet(), containsInAnyOrder("P1:logs*", "P2:metrics*")); assertThat(crossProjectRequest.getCanonicalExpressions().get("P1:logs*"), containsInAnyOrder("P1:logs*")); @@ -94,7 +94,7 @@ public void testQualifiedOnlyOriginProjectsRewrite() { IndicesOptions.DEFAULT ); - CPSExpressionRewriter.maybeRewriteCrossProjectResolvableRequest(remoteClusterAware, authorizedProjects, crossProjectRequest); + CrossProjectResolverUtils.maybeRewriteCrossProjectResolvableRequest(remoteClusterAware, authorizedProjects, crossProjectRequest); assertThat(crossProjectRequest.getCanonicalExpressions().keySet(), containsInAnyOrder("_origin:logs*", "_origin:metrics*")); assertThat(crossProjectRequest.getCanonicalExpressions().get("_origin:logs*"), containsInAnyOrder("logs*")); @@ -111,7 +111,7 @@ public void testQualifiedOriginAndLikedProjectsRewrite() { IndicesOptions.DEFAULT ); - CPSExpressionRewriter.maybeRewriteCrossProjectResolvableRequest(remoteClusterAware, authorizedProjects, crossProjectRequest); + CrossProjectResolverUtils.maybeRewriteCrossProjectResolvableRequest(remoteClusterAware, authorizedProjects, crossProjectRequest); assertThat(crossProjectRequest.getCanonicalExpressions().keySet(), containsInAnyOrder("P1:logs*", "_origin:metrics*")); assertThat(crossProjectRequest.getCanonicalExpressions().get("P1:logs*"), containsInAnyOrder("P1:logs*")); @@ -128,7 +128,7 @@ public void testQualifiedStartsWithProjectWildcardRewrite() { IndicesOptions.DEFAULT ); - CPSExpressionRewriter.maybeRewriteCrossProjectResolvableRequest(remoteClusterAware, authorizedProjects, crossProjectRequest); + CrossProjectResolverUtils.maybeRewriteCrossProjectResolvableRequest(remoteClusterAware, authorizedProjects, crossProjectRequest); assertThat(crossProjectRequest.getCanonicalExpressions().keySet(), containsInAnyOrder("P*:metrics*")); assertThat(crossProjectRequest.getCanonicalExpressions().get("P*:metrics*"), containsInAnyOrder("P1:metrics*", "P2:metrics*")); @@ -144,7 +144,7 @@ public void testQualifiedEndsWithProjectWildcardRewrite() { IndicesOptions.DEFAULT ); - CPSExpressionRewriter.maybeRewriteCrossProjectResolvableRequest(remoteClusterAware, authorizedProjects, crossProjectRequest); + CrossProjectResolverUtils.maybeRewriteCrossProjectResolvableRequest(remoteClusterAware, authorizedProjects, crossProjectRequest); assertThat(crossProjectRequest.getCanonicalExpressions().keySet(), containsInAnyOrder("*1:metrics*")); assertThat(crossProjectRequest.getCanonicalExpressions().get("*1:metrics*"), containsInAnyOrder("P1:metrics*", "Q1:metrics*")); @@ -160,7 +160,7 @@ public void testWildcardOnlyProjectRewrite() { IndicesOptions.DEFAULT ); - CPSExpressionRewriter.maybeRewriteCrossProjectResolvableRequest(remoteClusterAware, authorizedProjects, crossProjectRequest); + CrossProjectResolverUtils.maybeRewriteCrossProjectResolvableRequest(remoteClusterAware, authorizedProjects, crossProjectRequest); assertThat(crossProjectRequest.getCanonicalExpressions().keySet(), containsInAnyOrder("*:metrics*")); assertThat( @@ -169,6 +169,46 @@ public void testWildcardOnlyProjectRewrite() { ); } + public void testEmptyExpressionShouldMatchAll() { + AuthorizedProjectsSupplier.AuthorizedProjects authorizedProjects = new AuthorizedProjectsSupplier.AuthorizedProjects( + "_origin", + List.of("P1", "P2") + ); + CrossProjectReplaceableTest crossProjectRequest = new CrossProjectReplaceableTest(new String[] {}, IndicesOptions.DEFAULT); + + CrossProjectResolverUtils.maybeRewriteCrossProjectResolvableRequest(remoteClusterAware, authorizedProjects, crossProjectRequest); + + assertThat(crossProjectRequest.getCanonicalExpressions().keySet(), containsInAnyOrder("*")); + assertThat(crossProjectRequest.getCanonicalExpressions().get("*"), containsInAnyOrder("P1:*", "P2:*", "*")); + } + + public void testWildcardExpressionShouldMatchAll() { + AuthorizedProjectsSupplier.AuthorizedProjects authorizedProjects = new AuthorizedProjectsSupplier.AuthorizedProjects( + "_origin", + List.of("P1", "P2") + ); + CrossProjectReplaceableTest crossProjectRequest = new CrossProjectReplaceableTest(new String[] { "*" }, IndicesOptions.DEFAULT); + + CrossProjectResolverUtils.maybeRewriteCrossProjectResolvableRequest(remoteClusterAware, authorizedProjects, crossProjectRequest); + + assertThat(crossProjectRequest.getCanonicalExpressions().keySet(), containsInAnyOrder("*")); + assertThat(crossProjectRequest.getCanonicalExpressions().get("*"), containsInAnyOrder("P1:*", "P2:*", "*")); + } + + public void test_ALLExpressionShouldMatchAll() { + AuthorizedProjectsSupplier.AuthorizedProjects authorizedProjects = new AuthorizedProjectsSupplier.AuthorizedProjects( + "_origin", + List.of("P1", "P2") + ); + String all = randomBoolean() ? "_ALL" : "_all"; + CrossProjectReplaceableTest crossProjectRequest = new CrossProjectReplaceableTest(new String[] { all }, IndicesOptions.DEFAULT); + + CrossProjectResolverUtils.maybeRewriteCrossProjectResolvableRequest(remoteClusterAware, authorizedProjects, crossProjectRequest); + + assertThat(crossProjectRequest.getCanonicalExpressions().keySet(), containsInAnyOrder("*")); + assertThat(crossProjectRequest.getCanonicalExpressions().get("*"), containsInAnyOrder("P1:*", "P2:*", "*")); + } + private static class RemoteClusterAwareTest extends RemoteClusterAware { RemoteClusterAwareTest() { super(Settings.EMPTY); From b1f5fc9be46603669f29cc6ec40df4c43f986189 Mon Sep 17 00:00:00 2001 From: piergm <134913285+piergm@users.noreply.github.com> Date: Fri, 29 Aug 2025 16:56:08 +0200 Subject: [PATCH 6/6] handling of exclusions --- .../action/CrossProjectResolverUtils.java | 18 +++++++- .../CrossProjectResolverUtilsTests.java | 46 +++++++++++++++++++ 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/CrossProjectResolverUtils.java b/server/src/main/java/org/elasticsearch/action/CrossProjectResolverUtils.java index 85fd313626c38..b5f3da8eaffcd 100644 --- a/server/src/main/java/org/elasticsearch/action/CrossProjectResolverUtils.java +++ b/server/src/main/java/org/elasticsearch/action/CrossProjectResolverUtils.java @@ -28,6 +28,7 @@ public class CrossProjectResolverUtils { private static final Logger logger = LogManager.getLogger(CrossProjectResolverUtils.class); private static final String WILDCARD = "*"; private static final String MATCH_ALL = "_ALL"; + private static final String EXCLUSION = "-"; public static void maybeRewriteCrossProjectResolvableRequest( RemoteClusterAware remoteClusterAware, @@ -55,24 +56,37 @@ public static void maybeRewriteCrossProjectResolvableRequest( // handling of match all cases indices = new String[] { WILDCARD }; } + boolean atLeastOneResourceWasFound = true; Map> canonicalExpressionsMap = new LinkedHashMap<>(indices.length); for (String indexExpression : indices) { - // TODO we need to handle exclusions here already + // TODO We will need to handle exclusions here. For now we are throwing instead if we see an exclusion. + if (EXCLUSION.equals(indexExpression)) { + throw new IllegalArgumentException( + "Exclusions are not currently supported but was found in the expression [" + indexExpression + "]" + ); + } boolean isQualified = isQualifiedIndexExpression(indexExpression); if (isQualified) { // TODO handle empty case here -- empty means "search all" in ES which is _not_ what we want List canonicalExpressions = rewriteQualified(indexExpression, targetProjects, remoteClusterAware); // could fail early here in ignore_unavailable and allow_no_indices strict mode if things are empty canonicalExpressionsMap.put(indexExpression, canonicalExpressions); + if (canonicalExpressions.isEmpty() == false) { + atLeastOneResourceWasFound = false; + } logger.info("Rewrote qualified expression [{}] to [{}]", indexExpression, canonicalExpressions); } else { + atLeastOneResourceWasFound = false; // un-qualified expression, i.e. flat-world List canonicalExpressions = rewriteUnqualified(indexExpression, targetProjects.projects()); canonicalExpressionsMap.put(indexExpression, canonicalExpressions); logger.info("Rewrote unqualified expression [{}] to [{}]", indexExpression, canonicalExpressions); } } - + if (atLeastOneResourceWasFound) { + // Do we want to throw in this case? + throw new ResourceNotFoundException("no target projects for cross-project search request"); + } request.setCanonicalExpressions(canonicalExpressionsMap); } diff --git a/server/src/test/java/org/elasticsearch/action/CrossProjectResolverUtilsTests.java b/server/src/test/java/org/elasticsearch/action/CrossProjectResolverUtilsTests.java index 8d1e24d79a63e..abb1b7d831989 100644 --- a/server/src/test/java/org/elasticsearch/action/CrossProjectResolverUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/action/CrossProjectResolverUtilsTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.action; +import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.ESTestCase; @@ -150,6 +151,51 @@ public void testQualifiedEndsWithProjectWildcardRewrite() { assertThat(crossProjectRequest.getCanonicalExpressions().get("*1:metrics*"), containsInAnyOrder("P1:metrics*", "Q1:metrics*")); } + public void testProjectWildcardNotMatchingAnythingShouldThrow() { + AuthorizedProjectsSupplier.AuthorizedProjects authorizedProjects = new AuthorizedProjectsSupplier.AuthorizedProjects( + "_origin", + List.of("P1", "P2", "Q1", "Q2") + ); + CrossProjectReplaceableTest crossProjectRequest = new CrossProjectReplaceableTest( + new String[] { "S*:metrics*" }, + IndicesOptions.DEFAULT + ); + + // In this case we are throwing because no resource was found. Do we want to throw or should we continue with a list of empty + // "canonical" indices and perhaps throw later on based on IndicesOptions? + expectThrows( + ResourceNotFoundException.class, + () -> CrossProjectResolverUtils.maybeRewriteCrossProjectResolvableRequest( + remoteClusterAware, + authorizedProjects, + crossProjectRequest + ) + ); + } + + public void testRewritingShouldThrowOnIndexExclusions() { + // TODO Implement index exclusion. + AuthorizedProjectsSupplier.AuthorizedProjects authorizedProjects = new AuthorizedProjectsSupplier.AuthorizedProjects( + "_origin", + List.of("P1", "P2", "Q1", "Q2") + ); + CrossProjectReplaceableTest crossProjectRequest = new CrossProjectReplaceableTest( + new String[] { "P*:metrics*", "-P1:metrics*" }, + IndicesOptions.DEFAULT + ); + + // In this case we are throwing because no resource was found. Do we want to throw or should we continue with a list of empty + // "canonical" indices and perhaps throw later on based on IndicesOptions? + expectThrows( + IllegalArgumentException.class, + () -> CrossProjectResolverUtils.maybeRewriteCrossProjectResolvableRequest( + remoteClusterAware, + authorizedProjects, + crossProjectRequest + ) + ); + } + public void testWildcardOnlyProjectRewrite() { AuthorizedProjectsSupplier.AuthorizedProjects authorizedProjects = new AuthorizedProjectsSupplier.AuthorizedProjects( "_origin",