[CPS] Create Cross Project IndicesOptions (#135470)

An initial IndicesOptions for CrossProject.

This will be used to identify when the index must be resolved for a
cross-project search. It currently always defaults to false so we can
start reading the value, and a later commit will handle enabling it.

Author: prwhelan

server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java

@@ -159,7 +159,8 @@ protected ClusterBlockException checkBlock(RolloverRequest request, ClusterState
                 .matchOpen(request.indicesOptions().expandWildcardsOpen())
                 .matchClosed(request.indicesOptions().expandWildcardsClosed())
                 .build(),
-            IndicesOptions.GatekeeperOptions.DEFAULT
+            IndicesOptions.GatekeeperOptions.DEFAULT,
+            IndicesOptions.CrossProjectModeOptions.DEFAULT
         );
         ResolvedExpression resolvedRolloverTarget = SelectorResolver.parseExpression(request.getRolloverTarget(), request.indicesOptions());
         final IndexAbstraction indexAbstraction = projectMetadata.getIndicesLookup().get(resolvedRolloverTarget.resource());
@@ -255,7 +256,8 @@ protected void masterOperation(
         final var statsIndicesOptions = new IndicesOptions(
             IndicesOptions.ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS,
             IndicesOptions.WildcardOptions.builder().matchClosed(true).allowEmptyExpressions(false).build(),
-            IndicesOptions.GatekeeperOptions.DEFAULT
+            IndicesOptions.GatekeeperOptions.DEFAULT,
+            IndicesOptions.CrossProjectModeOptions.DEFAULT
         );
         // Make sure to recombine any selectors on the stats request
         IndicesStatsRequest statsRequest = new IndicesStatsRequest().indices(resolvedRolloverTarget.combined())

server/src/main/java/org/elasticsearch/action/support/IndicesOptions.java

@@ -9,9 +9,11 @@
 package org.elasticsearch.action.support;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.core.Nullable;
@@ -45,11 +47,14 @@
  * @param gatekeeperOptions, applies to all the resolved indices and defines if throttled will be included and if certain type of
  *                        aliases or indices are allowed, or they will throw an error. It acts as a gatekeeper when an action
  *                        does not support certain options.
+ * @param crossProjectModeOptions, applies to all the indices and adds logic specific for cross-project search. These options are
+ *                                 internal-only and can change over the lifetime of a single request.
  */
 public record IndicesOptions(
     ConcreteTargetOptions concreteTargetOptions,
     WildcardOptions wildcardOptions,
-    GatekeeperOptions gatekeeperOptions
+    GatekeeperOptions gatekeeperOptions,
+    CrossProjectModeOptions crossProjectModeOptions
 ) implements ToXContentFragment {
 
     public static IndicesOptions.Builder builder() {
@@ -413,6 +418,37 @@ public static Builder builder(GatekeeperOptions gatekeeperOptions) {
         }
     }
 
+    /**
+     * The cross-project mode options are internal-only options that apply on all indices that have been selected by the other Options.
+     * These options may contextually change over the lifetime of the request.
+     * @param resolveIndexExpression determines that the index expression must be resolved for cross-project requests, defaults to false.
+     */
+    public record CrossProjectModeOptions(boolean resolveIndexExpression) implements Writeable {
+
+        public static final CrossProjectModeOptions DEFAULT = new CrossProjectModeOptions(false);
+
+        private static final TransportVersion INDICES_OPTIONS_RESOLUTION_MODE = TransportVersion.fromName(
+            "indices_options_resolution_mode"
+        );
+
+        private static final String INDEX_EXPRESSION_NAME = "resolve_cross_project_index_expression";
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            if (out.getTransportVersion().supports(INDICES_OPTIONS_RESOLUTION_MODE)) {
+                out.writeBoolean(resolveIndexExpression);
+            }
+        }
+
+        public static CrossProjectModeOptions readFrom(StreamInput in) throws IOException {
+            if (in.getTransportVersion().supports(INDICES_OPTIONS_RESOLUTION_MODE)) {
+                return new CrossProjectModeOptions(in.readBoolean());
+            } else {
+                return CrossProjectModeOptions.DEFAULT;
+            }
+        }
+    }
+
     /**
      * This class is maintained for backwards compatibility and performance purposes. We use it for serialisation along with {@link Option}.
      */
@@ -463,7 +499,8 @@ private enum Option {
     public static final IndicesOptions DEFAULT = new IndicesOptions(
         ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS,
         WildcardOptions.DEFAULT,
-        GatekeeperOptions.DEFAULT
+        GatekeeperOptions.DEFAULT,
+        CrossProjectModeOptions.DEFAULT
     );
 
     public static final IndicesOptions STRICT_EXPAND_OPEN = IndicesOptions.builder()
@@ -857,6 +894,13 @@ public boolean ignoreThrottled() {
         return gatekeeperOptions().ignoreThrottled();
     }
 
+    /**
+     * @return whether indices will resolve to the cross-project "flat world" expression
+     */
+    public boolean resolveCrossProjectIndexExpression() {
+        return crossProjectModeOptions().resolveIndexExpression();
+    }
+
     public void writeIndicesOptions(StreamOutput out) throws IOException {
         EnumSet<Option> backwardsCompatibleOptions = EnumSet.noneOf(Option.class);
         if (allowNoIndices()) {
@@ -902,6 +946,7 @@ public void writeIndicesOptions(StreamOutput out) throws IOException {
             out.writeBoolean(true);
             out.writeBoolean(false);
         }
+        out.writeWriteable(crossProjectModeOptions);
     }
 
     public static IndicesOptions readIndicesOptions(StreamInput in) throws IOException {
@@ -930,14 +975,16 @@ public static IndicesOptions readIndicesOptions(StreamInput in) throws IOExcepti
                 ? ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS
                 : ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS,
             wildcardOptions,
-            gatekeeperOptions
+            gatekeeperOptions,
+            CrossProjectModeOptions.readFrom(in)
         );
     }
 
     public static class Builder {
         private ConcreteTargetOptions concreteTargetOptions;
         private WildcardOptions wildcardOptions;
         private GatekeeperOptions gatekeeperOptions;
+        private CrossProjectModeOptions crossProjectModeOptions;
 
         Builder() {
             this(DEFAULT);
@@ -947,6 +994,7 @@ public static class Builder {
             concreteTargetOptions = indicesOptions.concreteTargetOptions;
             wildcardOptions = indicesOptions.wildcardOptions;
             gatekeeperOptions = indicesOptions.gatekeeperOptions;
+            crossProjectModeOptions = indicesOptions.crossProjectModeOptions;
         }
 
         public Builder concreteTargetOptions(ConcreteTargetOptions concreteTargetOptions) {
@@ -974,8 +1022,13 @@ public Builder gatekeeperOptions(GatekeeperOptions.Builder generalOptions) {
             return this;
         }
 
+        public Builder crossProjectModeOptions(CrossProjectModeOptions crossProjectModeOptions) {
+            this.crossProjectModeOptions = crossProjectModeOptions;
+            return this;
+        }
+
         public IndicesOptions build() {
-            return new IndicesOptions(concreteTargetOptions, wildcardOptions, gatekeeperOptions);
+            return new IndicesOptions(concreteTargetOptions, wildcardOptions, gatekeeperOptions, crossProjectModeOptions);
         }
     }
 
@@ -1077,7 +1130,8 @@ public static IndicesOptions fromOptions(
         return new IndicesOptions(
             ignoreUnavailable ? ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS : ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS,
             wildcards,
-            gatekeeperOptions
+            gatekeeperOptions,
+            CrossProjectModeOptions.DEFAULT
         );
     }
 
@@ -1148,14 +1202,18 @@ public static IndicesOptions fromParameters(
             return defaultSettings;
         }
 
-        WildcardOptions wildcards = WildcardOptions.parseParameters(wildcardsString, allowNoIndicesString, defaultSettings.wildcardOptions);
-        GatekeeperOptions gatekeeperOptions = GatekeeperOptions.parseParameter(ignoreThrottled, defaultSettings.gatekeeperOptions);
+        var wildcards = WildcardOptions.parseParameters(wildcardsString, allowNoIndicesString, defaultSettings.wildcardOptions);
+        var gatekeeperOptions = GatekeeperOptions.parseParameter(ignoreThrottled, defaultSettings.gatekeeperOptions);
+        var crossProjectModeOptions = defaultSettings.crossProjectModeOptions != null
+            ? defaultSettings.crossProjectModeOptions
+            : CrossProjectModeOptions.DEFAULT;
 
         // note that allowAliasesToMultipleIndices is not exposed, always true (only for internal use)
         return IndicesOptions.builder()
             .concreteTargetOptions(ConcreteTargetOptions.fromParameter(ignoreUnavailableString, defaultSettings.concreteTargetOptions))
             .wildcardOptions(wildcards)
             .gatekeeperOptions(gatekeeperOptions)
+            .crossProjectModeOptions(crossProjectModeOptions)
             .build();
     }
 
@@ -1421,6 +1479,8 @@ public String toString() {
             + allowSelectors()
             + ", include_failure_indices="
             + includeFailureIndices()
+            + ", resolve_cross_project_index_expression="
+            + resolveCrossProjectIndexExpression()
             + ']';
     }
 }

server/src/main/resources/transport/definitions/referable/indices_options_resolution_mode.csv

@@ -0,0 +1 @@
+9192000

server/src/main/resources/transport/upper_bounds/9.3.csv

@@ -1 +1 @@
-add_cross_cluster_api_key_signature,9191000
+indices_options_resolution_mode,9192000

server/src/test/java/org/elasticsearch/action/support/IndicesOptionsTests.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.action.support;
 
 import org.elasticsearch.action.support.IndicesOptions.ConcreteTargetOptions;
+import org.elasticsearch.action.support.IndicesOptions.CrossProjectModeOptions;
 import org.elasticsearch.action.support.IndicesOptions.GatekeeperOptions;
 import org.elasticsearch.action.support.IndicesOptions.WildcardOptions;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -57,6 +58,7 @@ public void testSerialization() throws Exception {
                         .allowClosedIndices(randomBoolean())
                         .allowSelectors(randomBoolean())
                 )
+                .crossProjectModeOptions(new CrossProjectModeOptions(randomBoolean()))
                 .build();
 
             BytesStreamOutput output = new BytesStreamOutput();
@@ -102,6 +104,7 @@ public void testFromOptions() {
         assertThat(indicesOptions.forbidClosedIndices(), equalTo(forbidClosedIndices));
         assertEquals(ignoreAliases, indicesOptions.ignoreAliases());
         assertEquals(ignoreThrottled, indicesOptions.ignoreThrottled());
+        assertEquals(indicesOptions.resolveCrossProjectIndexExpression(), CrossProjectModeOptions.DEFAULT.resolveIndexExpression());
     }
 
     public void testFromOptionsWithDefaultOptions() {
@@ -138,6 +141,7 @@ public void testFromOptionsWithDefaultOptions() {
         assertEquals(defaultOptions.allowAliasesToMultipleIndices(), indicesOptions.allowAliasesToMultipleIndices());
         assertEquals(defaultOptions.forbidClosedIndices(), indicesOptions.forbidClosedIndices());
         assertEquals(defaultOptions.ignoreAliases(), indicesOptions.ignoreAliases());
+        assertEquals(defaultOptions.resolveCrossProjectIndexExpression(), indicesOptions.resolveCrossProjectIndexExpression());
     }
 
     public void testFromParameters() {
@@ -201,6 +205,7 @@ public void testFromParameters() {
         assertEquals(defaultOptions.allowAliasesToMultipleIndices(), updatedOptions.allowAliasesToMultipleIndices());
         assertEquals(defaultOptions.forbidClosedIndices(), updatedOptions.forbidClosedIndices());
         assertEquals(defaultOptions.ignoreAliases(), updatedOptions.ignoreAliases());
+        assertEquals(defaultOptions.resolveCrossProjectIndexExpression(), updatedOptions.resolveCrossProjectIndexExpression());
     }
 
     public void testEqualityAndHashCode() {
@@ -330,6 +335,7 @@ public void testFromMap() {
         assertEquals(ignoreUnavailable == null ? defaults.ignoreUnavailable() : ignoreUnavailable, fromMap.ignoreUnavailable());
         assertEquals(allowNoIndices == null ? defaults.allowNoIndices() : allowNoIndices, fromMap.allowNoIndices());
         assertEquals(ignoreThrottled == null ? defaults.ignoreThrottled() : ignoreThrottled, fromMap.ignoreThrottled());
+        assertEquals(fromMap.resolveCrossProjectIndexExpression(), CrossProjectModeOptions.DEFAULT.resolveIndexExpression());
     }
 
     public void testToXContent() throws IOException {
@@ -348,8 +354,14 @@ public void testToXContent() throws IOException {
             randomBoolean(),
             randomBoolean()
         );
+        CrossProjectModeOptions crossProjectModeOptions = new CrossProjectModeOptions(randomBoolean());
 
-        IndicesOptions indicesOptions = new IndicesOptions(concreteTargetOptions, wildcardOptions, gatekeeperOptions);
+        IndicesOptions indicesOptions = new IndicesOptions(
+            concreteTargetOptions,
+            wildcardOptions,
+            gatekeeperOptions,
+            crossProjectModeOptions
+        );
 
         XContentType type = randomFrom(XContentType.values());
         BytesReference xContentBytes = toXContentBytes(indicesOptions, type);
@@ -423,6 +435,7 @@ public void testFromXContentWithWildcardSpecialValues() throws IOException {
         assertTrue(fromXContentOptions.expandWildcardsClosed());
         assertTrue(fromXContentOptions.expandWildcardsHidden());
         assertTrue(fromXContentOptions.expandWildcardsOpen());
+        assertFalse(fromXContentOptions.resolveCrossProjectIndexExpression());
 
         try (XContentBuilder builder = XContentFactory.contentBuilder(type)) {
             builder.startObject();
@@ -441,6 +454,7 @@ public void testFromXContentWithWildcardSpecialValues() throws IOException {
         assertFalse(fromXContentOptions.expandWildcardsClosed());
         assertFalse(fromXContentOptions.expandWildcardsHidden());
         assertFalse(fromXContentOptions.expandWildcardsOpen());
+        assertFalse(fromXContentOptions.resolveCrossProjectIndexExpression());
     }
 
     public void testFromXContentWithDefaults() throws Exception {
@@ -494,6 +508,7 @@ public void testFromXContentWithDefaults() throws Exception {
         }
         assertEquals(ignoreUnavailable, fromXContentOptions.ignoreUnavailable());
         assertEquals(expectedWildcardStates, fromXContentOptions.wildcardOptions());
+        assertFalse(fromXContentOptions.resolveCrossProjectIndexExpression());
     }
 
     private BytesReference toXContentBytes(IndicesOptions indicesOptions, XContentType type) throws IOException {

server/src/test/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolverTests.java

@@ -2371,7 +2371,8 @@ public void testIgnoreThrottled() {
                 new IndicesOptions(
                     IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS,
                     IndicesOptions.WildcardOptions.DEFAULT,
-                    IndicesOptions.GatekeeperOptions.builder().ignoreThrottled(true).build()
+                    IndicesOptions.GatekeeperOptions.builder().ignoreThrottled(true).build(),
+                    IndicesOptions.CrossProjectModeOptions.DEFAULT
                 ),
                 "ind*",
                 "test-index"

x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java

@@ -349,7 +349,8 @@ public void testIndexDoesntExist() {
                         + "indices_options [IndicesOptions[ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, "
                         + "expand_wildcards_closed=false, expand_wildcards_hidden=false, allow_aliases_to_multiple_indices=true, "
                         + "forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true, "
-                        + "allow_selectors=true, include_failure_indices=false]] with exception [no such index [not_foo]]"
+                        + "allow_selectors=true, include_failure_indices=false, resolve_cross_project_index_expression=false]] "
+                        + "with exception [no such index [not_foo]]"
                 )
             )
         );
@@ -383,8 +384,8 @@ public void testIndexDoesntExist() {
                         + "indices given [not_foo] and indices_options [IndicesOptions[ignore_unavailable=false, allow_no_indices=true, "
                         + "expand_wildcards_open=true, expand_wildcards_closed=false, expand_wildcards_hidden=false, "
                         + "allow_aliases_to_multiple_indices=true, forbid_closed_indices=true, ignore_aliases=false, "
-                        + "ignore_throttled=true, allow_selectors=true, include_failure_indices=false]] with exception "
-                        + "[no such index [not_foo]]]"
+                        + "ignore_throttled=true, allow_selectors=true, include_failure_indices=false,"
+                        + " resolve_cross_project_index_expression=false]] with exception [no such index [not_foo]]]"
                 )
             )
         );
@@ -561,7 +562,8 @@ public void testSelectNode_GivenJobOpeningAndIndexDoesNotExist() {
                         + "indices_options [IndicesOptions[ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, "
                         + "expand_wildcards_closed=false, expand_wildcards_hidden=false, allow_aliases_to_multiple_indices=true, "
                         + "forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true, "
-                        + "allow_selectors=true, include_failure_indices=false]] with exception [no such index [not_foo]]]"
+                        + "allow_selectors=true, include_failure_indices=false, resolve_cross_project_index_expression=false]]"
+                        + " with exception [no such index [not_foo]]]"
                 )
             )
         );