Merge branch 'main' of github.com:elastic/elasticsearch into additional-migration-info-for-ds-reindex

Author: lukewhiting

docs/changelog/117581.yaml

@@ -0,0 +1,5 @@
+pr: 117581
+summary: Make reserved built-in roles queryable
+area: Authorization
+type: enhancement
+issues: []

modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java

@@ -43,7 +43,6 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.index.IndexVersion;
@@ -80,28 +79,19 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
-import java.util.function.BiConsumer;
 import java.util.function.Supplier;
 
 import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
 public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBuilder> {
-    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(ParseField.class);
-    static final String DOCUMENT_TYPE_DEPRECATION_MESSAGE = "[types removal] Types are deprecated in [percolate] queries. "
-        + "The [document_type] should no longer be specified.";
-    static final String TYPE_DEPRECATION_MESSAGE = "[types removal] Types are deprecated in [percolate] queries. "
-        + "The [type] of the indexed document should no longer be specified.";
-
     public static final String NAME = "percolate";
 
     static final ParseField DOCUMENT_FIELD = new ParseField("document");
     static final ParseField DOCUMENTS_FIELD = new ParseField("documents");
     private static final ParseField NAME_FIELD = new ParseField("name");
     private static final ParseField QUERY_FIELD = new ParseField("field");
-    private static final ParseField DOCUMENT_TYPE_FIELD = new ParseField("document_type");
-    private static final ParseField INDEXED_DOCUMENT_FIELD_TYPE = new ParseField("type");
     private static final ParseField INDEXED_DOCUMENT_FIELD_INDEX = new ParseField("index");
     private static final ParseField INDEXED_DOCUMENT_FIELD_ID = new ParseField("id");
     private static final ParseField INDEXED_DOCUMENT_FIELD_ROUTING = new ParseField("routing");
@@ -368,10 +358,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
         );
     }
 
-    private static BiConsumer<PercolateQueryBuilder, String> deprecateAndIgnoreType(String key, String message) {
-        return (target, type) -> deprecationLogger.compatibleCritical(key, message);
-    }
-
     private static BytesReference parseDocument(XContentParser parser) throws IOException {
         try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
             builder.copyCurrentStructure(parser);

muted-tests.yml

@@ -40,9 +40,6 @@ tests:
 - class: org.elasticsearch.packaging.test.WindowsServiceTests
   method: test33JavaChanged
   issue: https://github.com/elastic/elasticsearch/issues/113177
-- class: org.elasticsearch.smoketest.MlWithSecurityIT
-  method: test {yaml=ml/sparse_vector_search/Test sparse_vector search with query vector and pruning config}
-  issue: https://github.com/elastic/elasticsearch/issues/108997
 - class: org.elasticsearch.packaging.test.WindowsServiceTests
   method: test80JavaOptsInEnvVar
   issue: https://github.com/elastic/elasticsearch/issues/113219
@@ -159,9 +156,6 @@ tests:
   issue: https://github.com/elastic/elasticsearch/issues/117473
 - class: org.elasticsearch.repositories.s3.RepositoryS3EcsClientYamlTestSuiteIT
   issue: https://github.com/elastic/elasticsearch/issues/117525
-- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
-  method: test {p0=synonyms/90_synonyms_reloading_for_synset/Reload analyzers for specific synonym set}
-  issue: https://github.com/elastic/elasticsearch/issues/116777
 - class: "org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT"
   method: "test {scoring.*}"
   issue: https://github.com/elastic/elasticsearch/issues/117641
@@ -302,6 +296,14 @@ tests:
 - class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests
   method: testInvalidJSON
   issue: https://github.com/elastic/elasticsearch/issues/116521
+- class: org.elasticsearch.xpack.ccr.rest.ShardChangesRestIT
+  method: testShardChangesNoOperation
+  issue: https://github.com/elastic/elasticsearch/issues/118800
+- class: org.elasticsearch.xpack.security.QueryableReservedRolesIT
+  method: testDeletingAndCreatingSecurityIndexTriggersSynchronization
+  issue: https://github.com/elastic/elasticsearch/issues/118806
+- class: org.elasticsearch.xpack.esql.session.IndexResolverFieldNamesTests
+  issue: https://github.com/elastic/elasticsearch/issues/118814
 
 # Examples:
 #

rest-api-spec/build.gradle

@@ -69,4 +69,5 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task ->
   task.skipTest("search/520_fetch_fields/fetch _seq_no via fields", "error code is changed from 5xx to 400 in 9.0")
   task.skipTest("search.vectors/41_knn_search_bbq_hnsw/Test knn search", "Scoring has changed in latest versions")
   task.skipTest("search.vectors/42_knn_search_bbq_flat/Test knn search", "Scoring has changed in latest versions")
+  task.skipTest("synonyms/90_synonyms_reloading_for_synset/Reload analyzers for specific synonym set", "Can't work until auto-expand replicas is 0-1 for synonyms index")
 })

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/90_synonyms_reloading_for_synset.yml

@@ -1,8 +1,8 @@
----
-"Reload analyzers for specific synonym set":
+setup:
   - requires:
       cluster_features: ["gte_v8.10.0"]
       reason: Reloading analyzers for specific synonym set is introduced in 8.10.0
+
   # Create synonyms_set1
   - do:
       synonyms.put_synonym:
@@ -100,21 +100,25 @@
           - '{"index": {"_index": "my_index2", "_id": "2"}}'
           - '{"my_field": "goodbye"}'
 
-  # An update of synonyms_set1 must trigger auto-reloading of analyzers only for synonyms_set1
+---
+"Reload analyzers for specific synonym set":
+# These specific tests can't succeed in BwC, as synonyms auto-expand replicas are 0-all. Replicas can't be associated to
+#  upgraded nodes, and thus we are not able to guarantee that the shards are not failed.
+# This test is skipped for BwC until synonyms index has auto-exapnd replicas set to 0-1.
+
   - do:
       synonyms.put_synonym:
         id: synonyms_set1
         body:
           synonyms_set:
             - synonyms: "hello, salute"
             - synonyms: "ciao => goodbye"
+
   - match: { result: "updated" }
   - gt: { reload_analyzers_details._shards.total: 0 }
   - gt: { reload_analyzers_details._shards.successful: 0 }
   - match: { reload_analyzers_details._shards.failed: 0 }
-  - length: { reload_analyzers_details.reload_details: 1 } # reload details contain only a single index
-  - match: { reload_analyzers_details.reload_details.0.index: "my_index1" }
-  - match: { reload_analyzers_details.reload_details.0.reloaded_analyzers.0: "my_analyzer1" }
+
 
   # Confirm that the index analyzers are reloaded for my_index1
   - do:
@@ -127,6 +131,23 @@
                 query: salute
   - match: { hits.total.value: 1 }
 
+---
+"Check analyzer reloaded and non failed shards for bwc tests":
+
+  - do:
+      synonyms.put_synonym:
+        id: synonyms_set1
+        body:
+          synonyms_set:
+            - synonyms: "hello, salute"
+            - synonyms: "ciao => goodbye"
+  - match: { result: "updated" }
+  - gt: { reload_analyzers_details._shards.total: 0 }
+  - gt: { reload_analyzers_details._shards.successful: 0 }
+  - length: { reload_analyzers_details.reload_details: 1 } # reload details contain only a single index
+  - match: { reload_analyzers_details.reload_details.0.index: "my_index1" }
+  - match: { reload_analyzers_details.reload_details.0.reloaded_analyzers.0: "my_analyzer1" }
+
   # Confirm that the index analyzers are still the same for my_index2
   - do:
       search:

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -1138,7 +1138,7 @@ protected static void wipeAllIndices(boolean preserveSecurityIndices) throws IOE
         }
     }
 
-    private static boolean ignoreSystemIndexAccessWarnings(List<String> warnings) {
+    protected static boolean ignoreSystemIndexAccessWarnings(List<String> warnings) {
         for (String warning : warnings) {
             if (warning.startsWith("this request accesses system indices:")) {
                 SUITE_LOGGER.warn("Ignoring system index access warning during test cleanup: {}", warning);

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/ParsingException.java

@@ -11,7 +11,6 @@
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
-import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MultiMatchQueryPredicate;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or;
@@ -22,7 +21,6 @@
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardLike;
 import org.elasticsearch.xpack.esql.core.querydsl.query.BoolQuery;
 import org.elasticsearch.xpack.esql.core.querydsl.query.ExistsQuery;
-import org.elasticsearch.xpack.esql.core.querydsl.query.MultiMatchQuery;
 import org.elasticsearch.xpack.esql.core.querydsl.query.NotQuery;
 import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
 import org.elasticsearch.xpack.esql.core.querydsl.query.RegexQuery;
@@ -71,18 +69,6 @@ private static Query translateField(RegexMatch e, String targetFieldName) {
         }
     }
 
-    public static class MultiMatches extends ExpressionTranslator<MultiMatchQueryPredicate> {
-
-        @Override
-        protected Query asQuery(MultiMatchQueryPredicate q, TranslatorHandler handler) {
-            return doTranslate(q, handler);
-        }
-
-        public static Query doTranslate(MultiMatchQueryPredicate q, TranslatorHandler handler) {
-            return new MultiMatchQuery(q.source(), q.query(), q.fields(), q);
-        }
-    }
-
     public static class BinaryLogic extends ExpressionTranslator<
         org.elasticsearch.xpack.esql.core.expression.predicate.logical.BinaryLogic> {
 

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java

@@ -21,7 +21,7 @@
 import java.util.List;
 
 import static org.elasticsearch.xpack.esql.CsvTestUtils.isEnabled;
-import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V5;
+import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V6;
 import static org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase.Mode.ASYNC;
 
 public class MixedClusterEsqlSpecIT extends EsqlSpecTestCase {
@@ -96,7 +96,7 @@ protected boolean supportsInferenceTestService() {
 
     @Override
     protected boolean supportsIndexModeLookup() throws IOException {
-        return hasCapabilities(List.of(JOIN_LOOKUP_V5.capabilityName()));
+        return hasCapabilities(List.of(JOIN_LOOKUP_V6.capabilityName()));
     }
 
     @Override

x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java

@@ -48,7 +48,7 @@
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources;
 import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS;
 import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS_V2;
-import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V5;
+import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V6;
 import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_PLANNING_V1;
 import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.METADATA_FIELDS_REMOTE_TEST;
 import static org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase.Mode.SYNC;
@@ -124,7 +124,7 @@ protected void shouldSkipTest(String testName) throws IOException {
         assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS.capabilityName()));
         assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V2.capabilityName()));
         assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_PLANNING_V1.capabilityName()));
-        assumeFalse("LOOKUP JOIN not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_LOOKUP_V5.capabilityName()));
+        assumeFalse("LOOKUP JOIN not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_LOOKUP_V6.capabilityName()));
     }
 
     private TestFeatureService remoteFeaturesService() throws IOException {
@@ -283,8 +283,8 @@ protected boolean supportsInferenceTestService() {
 
     @Override
     protected boolean supportsIndexModeLookup() throws IOException {
-        // CCS does not yet support JOIN_LOOKUP_V5 and clusters falsely report they have this capability
-        // return hasCapabilities(List.of(JOIN_LOOKUP_V5.capabilityName()));
+        // CCS does not yet support JOIN_LOOKUP_V6 and clusters falsely report they have this capability
+        // return hasCapabilities(List.of(JOIN_LOOKUP_V6.capabilityName()));
         return false;
     }
 }