Merge remote-tracking branch 'origin/main' into lucene_snapshot

Author: elasticsearchmachine

distribution/docker/src/docker/iron_bank/hardening_manifest.yaml

@@ -50,9 +50,12 @@ resources:
 
 # List of project maintainers
 maintainers:
-  - name: "Rory Hunter"
-    email: "[email protected]"
-    username: "rory"
+  - name: "Mark Vieira"
+    email: "[email protected]"
+    username: "mark-vieira"
+  - name: "Rene Gröschke"
+    email: "[email protected]"
+    username: "breskeby"
   - email: "[email protected]"
     name: "Alexander Klepal"
     username: "alexander.klepal"

docs/changelog/117581.yaml

@@ -0,0 +1,5 @@
+pr: 117581
+summary: Make reserved built-in roles queryable
+area: Authorization
+type: enhancement
+issues: []

docs/changelog/117643.yaml

@@ -0,0 +1,6 @@
+pr: 117643
+summary: Drop null columns in text formats
+area: ES|QL
+type: bug
+issues:
+  - 116848

docs/changelog/117851.yaml

@@ -0,0 +1,5 @@
+pr: 117851
+summary: Addition of `tier_preference`, `creation_date` and `version` fields in Elasticsearch monitoring template
+area: Monitoring
+type: enhancement
+issues: []

docs/changelog/118454.yaml

@@ -0,0 +1,5 @@
+pr: 118454
+summary: Fix RLIKE folding with (unsupported) case insensitive pattern
+area: ES|QL
+type: bug
+issues: []

modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java

@@ -43,7 +43,6 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.index.IndexVersion;
@@ -80,28 +79,19 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
-import java.util.function.BiConsumer;
 import java.util.function.Supplier;
 
 import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
 public class PercolateQueryBuilder extends AbstractQueryBuilder<PercolateQueryBuilder> {
-    private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(ParseField.class);
-    static final String DOCUMENT_TYPE_DEPRECATION_MESSAGE = "[types removal] Types are deprecated in [percolate] queries. "
-        + "The [document_type] should no longer be specified.";
-    static final String TYPE_DEPRECATION_MESSAGE = "[types removal] Types are deprecated in [percolate] queries. "
-        + "The [type] of the indexed document should no longer be specified.";
-
     public static final String NAME = "percolate";
 
     static final ParseField DOCUMENT_FIELD = new ParseField("document");
     static final ParseField DOCUMENTS_FIELD = new ParseField("documents");
     private static final ParseField NAME_FIELD = new ParseField("name");
     private static final ParseField QUERY_FIELD = new ParseField("field");
-    private static final ParseField DOCUMENT_TYPE_FIELD = new ParseField("document_type");
-    private static final ParseField INDEXED_DOCUMENT_FIELD_TYPE = new ParseField("type");
     private static final ParseField INDEXED_DOCUMENT_FIELD_INDEX = new ParseField("index");
     private static final ParseField INDEXED_DOCUMENT_FIELD_ID = new ParseField("id");
     private static final ParseField INDEXED_DOCUMENT_FIELD_ROUTING = new ParseField("routing");
@@ -368,10 +358,6 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep
         );
     }
 
-    private static BiConsumer<PercolateQueryBuilder, String> deprecateAndIgnoreType(String key, String message) {
-        return (target, type) -> deprecationLogger.compatibleCritical(key, message);
-    }
-
     private static BytesReference parseDocument(XContentParser parser) throws IOException {
         try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
             builder.copyCurrentStructure(parser);

muted-tests.yml

@@ -40,9 +40,6 @@ tests:
 - class: org.elasticsearch.packaging.test.WindowsServiceTests
   method: test33JavaChanged
   issue: https://github.com/elastic/elasticsearch/issues/113177
-- class: org.elasticsearch.smoketest.MlWithSecurityIT
-  method: test {yaml=ml/sparse_vector_search/Test sparse_vector search with query vector and pruning config}
-  issue: https://github.com/elastic/elasticsearch/issues/108997
 - class: org.elasticsearch.packaging.test.WindowsServiceTests
   method: test80JavaOptsInEnvVar
   issue: https://github.com/elastic/elasticsearch/issues/113219
@@ -159,9 +156,6 @@ tests:
   issue: https://github.com/elastic/elasticsearch/issues/117473
 - class: org.elasticsearch.repositories.s3.RepositoryS3EcsClientYamlTestSuiteIT
   issue: https://github.com/elastic/elasticsearch/issues/117525
-- class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
-  method: test {p0=synonyms/90_synonyms_reloading_for_synset/Reload analyzers for specific synonym set}
-  issue: https://github.com/elastic/elasticsearch/issues/116777
 - class: "org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT"
   method: "test {scoring.*}"
   issue: https://github.com/elastic/elasticsearch/issues/117641
@@ -302,6 +296,14 @@ tests:
 - class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests
   method: testInvalidJSON
   issue: https://github.com/elastic/elasticsearch/issues/116521
+- class: org.elasticsearch.xpack.ccr.rest.ShardChangesRestIT
+  method: testShardChangesNoOperation
+  issue: https://github.com/elastic/elasticsearch/issues/118800
+- class: org.elasticsearch.xpack.security.QueryableReservedRolesIT
+  method: testDeletingAndCreatingSecurityIndexTriggersSynchronization
+  issue: https://github.com/elastic/elasticsearch/issues/118806
+- class: org.elasticsearch.xpack.esql.session.IndexResolverFieldNamesTests
+  issue: https://github.com/elastic/elasticsearch/issues/118814
 
 # Examples:
 #

rest-api-spec/build.gradle

@@ -71,4 +71,5 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task ->
   task.skipTest("search/520_fetch_fields/fetch _seq_no via fields", "error code is changed from 5xx to 400 in 9.0")
   task.skipTest("search.vectors/41_knn_search_bbq_hnsw/Test knn search", "Scoring has changed in latest versions")
   task.skipTest("search.vectors/42_knn_search_bbq_flat/Test knn search", "Scoring has changed in latest versions")
+  task.skipTest("synonyms/90_synonyms_reloading_for_synset/Reload analyzers for specific synonym set", "Can't work until auto-expand replicas is 0-1 for synonyms index")
 })

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/synonyms/90_synonyms_reloading_for_synset.yml

@@ -1,8 +1,8 @@
----
-"Reload analyzers for specific synonym set":
+setup:
   - requires:
       cluster_features: ["gte_v8.10.0"]
       reason: Reloading analyzers for specific synonym set is introduced in 8.10.0
+
   # Create synonyms_set1
   - do:
       synonyms.put_synonym:
@@ -100,21 +100,25 @@
           - '{"index": {"_index": "my_index2", "_id": "2"}}'
           - '{"my_field": "goodbye"}'
 
-  # An update of synonyms_set1 must trigger auto-reloading of analyzers only for synonyms_set1
+---
+"Reload analyzers for specific synonym set":
+# These specific tests can't succeed in BwC, as synonyms auto-expand replicas are 0-all. Replicas can't be associated to
+#  upgraded nodes, and thus we are not able to guarantee that the shards are not failed.
+# This test is skipped for BwC until synonyms index has auto-exapnd replicas set to 0-1.
+
   - do:
       synonyms.put_synonym:
         id: synonyms_set1
         body:
           synonyms_set:
             - synonyms: "hello, salute"
             - synonyms: "ciao => goodbye"
+
   - match: { result: "updated" }
   - gt: { reload_analyzers_details._shards.total: 0 }
   - gt: { reload_analyzers_details._shards.successful: 0 }
   - match: { reload_analyzers_details._shards.failed: 0 }
-  - length: { reload_analyzers_details.reload_details: 1 } # reload details contain only a single index
-  - match: { reload_analyzers_details.reload_details.0.index: "my_index1" }
-  - match: { reload_analyzers_details.reload_details.0.reloaded_analyzers.0: "my_analyzer1" }
+
 
   # Confirm that the index analyzers are reloaded for my_index1
   - do:
@@ -127,6 +131,23 @@
                 query: salute
   - match: { hits.total.value: 1 }
 
+---
+"Check analyzer reloaded and non failed shards for bwc tests":
+
+  - do:
+      synonyms.put_synonym:
+        id: synonyms_set1
+        body:
+          synonyms_set:
+            - synonyms: "hello, salute"
+            - synonyms: "ciao => goodbye"
+  - match: { result: "updated" }
+  - gt: { reload_analyzers_details._shards.total: 0 }
+  - gt: { reload_analyzers_details._shards.successful: 0 }
+  - length: { reload_analyzers_details.reload_details: 1 } # reload details contain only a single index
+  - match: { reload_analyzers_details.reload_details.0.index: "my_index1" }
+  - match: { reload_analyzers_details.reload_details.0.reloaded_analyzers.0: "my_analyzer1" }
+
   # Confirm that the index analyzers are still the same for my_index2
   - do:
       search:

server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java

@@ -48,7 +48,7 @@ public final class NoOpEngine extends ReadOnlyEngine {
     public NoOpEngine(EngineConfig config) {
         this(
             config,
-            config.isPromotableToPrimary() ? null : new TranslogStats(0, 0, 0, 0, 0),
+            config.isPromotableToPrimary() && config.getTranslogConfig().hasTranslog() ? null : new TranslogStats(0, 0, 0, 0, 0),
             config.isPromotableToPrimary()
                 ? null
                 : new SeqNoStats(