Merge branch 'main' into enhancement/126526

Author: prwhelan

docs/changelog/126385.yaml

@@ -0,0 +1,6 @@
+pr: 126385
+summary: Filter out empty top docs results before merging
+area: Search
+type: bug
+issues:
+ - 126118

docs/changelog/126550.yaml

@@ -0,0 +1,6 @@
+pr: 126550
+summary: Add leniency to missing array values in mustache
+area: Infra/Scripting
+type: bug
+issues:
+ - 55200

docs/release-notes/index.md

@@ -25,29 +25,25 @@ To check for security updates, go to [Security announcements for the Elastic sta
 
 ### Highlights [elasticsearch-900-highlights]
 
-::::{dropdown} Add new experimental `rank_vectors` mapping for late-interaction second order ranking
-Late-interaction models are powerful rerankers. While their size and overall cost doesn't lend itself for HNSW indexing, utilizing them as second order reranking can provide excellent boosts in relevance. The new `rank_vectors` mapping allows for rescoring over new and novel multi-vector late-interaction models like ColBERT or ColPali.
+::::{dropdown} rank_vectors field type is now available for late-interaction ranking
 
-For more information, check [PR #118804](https://github.com/elastic/elasticsearch/pull/118804).
-::::
+[`rank_vectors`](../reference/elasticsearch/mapping-reference/rank-vectors.md) is a new field type released as an experimental feature in Elasticsearch 9.0. It is designed to be used with dense vectors and allows for late-interaction second order ranking.
 
-::::{dropdown} Enable LOOKUP JOIN in non-snapshot builds
-This effectively releases LOOKUP JOIN into tech preview. Docs will
-follow in a separate PR.
+Late-interaction models are powerful rerankers. While their size and overall cost doesn’t lend itself for HNSW indexing, utilizing them as second order reranking can provide excellent boosts in relevance. The new `rank_vectors` mapping allows for rescoring over new and novel multi-vector late-interaction models like ColBERT or ColPali.
+::::
 
-- Enable the lexing/grammar for LOOKUP JOIN in non-snapshot builds.
-- Remove the grammar for the unsupported `| JOIN ...` command (without `LOOKUP` as first keyword). The way the lexer modes work, otherwise we'd also have to enable `| JOIN ...` syntax on non-snapshot builds and would have to add additional validation to provide appropriate error messages.
-- Remove grammar for `LOOKUP JOIN index AS ...` because qualifiers are not yet supported. Otherwise we'd have to put in additional validation as well to prevent such queries.
+::::{dropdown} ES|QL LOOKUP JOIN is now available in technical preview
 
-Also fix https://github.com/elastic/elasticsearch/issues/121185
+[LOOKUP JOIN](../reference/query-languages/esql/esql-commands.md) is now available in technical preview. LOOKUP JOIN combines data from your ES|QL queries with matching records from a lookup index, enabling you to:
 
-For more information, check [PR #121193](https://github.com/elastic/elasticsearch/pull/121193).
+- Enrich your search results with reference data
+- Speed up root-cause analysis and security investigations
+- Join data across indices without complex queries
+- Reduce operational overhead when correlating events
 ::::
 
-::::{dropdown} Release semantic_text as a GA feature
-semantic_text is now an official GA (generally available) feature! This field type allows you to easily set up and perform semantic search with minimal ramp up time.
-
-For more information, check [PR #124669](https://github.com/elastic/elasticsearch/pull/124669).
+::::{dropdown} The semantic_text field type is now GA
+[`semantic_text`](../reference/elasticsearch/mapping-reference/semantic-text.md) is now an official GA (generally available) feature! This field type allows you to easily set up and perform semantic search with minimal ramp up time.
 ::::
 
 ### Features and enhancements [elasticsearch-900-features-enhancements]

modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomReflectionObjectHandler.java

@@ -111,11 +111,11 @@ public Object get(Object key) {
             if ("size".equals(key)) {
                 return size();
             } else if (key instanceof Number number) {
-                return Array.get(array, number.intValue());
+                return number.intValue() >= 0 && number.intValue() < length ? Array.get(array, number.intValue()) : null;
             }
             try {
                 int index = Integer.parseInt(key.toString());
-                return Array.get(array, index);
+                return index >= 0 && index < length ? Array.get(array, index) : null;
             } catch (NumberFormatException nfe) {
                 // if it's not a number it is as if the key doesn't exist
                 return null;
@@ -169,11 +169,11 @@ public Object get(Object key) {
             if ("size".equals(key)) {
                 return col.size();
             } else if (key instanceof Number number) {
-                return Iterables.get(col, number.intValue());
+                return number.intValue() >= 0 && number.intValue() < col.size() ? Iterables.get(col, number.intValue()) : null;
             }
             try {
                 int index = Integer.parseInt(key.toString());
-                return Iterables.get(col, index);
+                return index >= 0 && index < col.size() ? Iterables.get(col, index) : null;
             } catch (NumberFormatException nfe) {
                 // if it's not a number it is as if the key doesn't exist
                 return null;

modules/lang-mustache/src/test/java/org/elasticsearch/script/mustache/MustacheTests.java

@@ -141,6 +141,26 @@ public void testMapInArrayAccess() throws Exception {
         assertThat(output, both(containsString("foo")).and(containsString("bar")));
     }
 
+    public void testMapInArrayBadAccess() throws Exception {
+        String template = "{{data.0.key}} {{data.2.key}}";
+        TemplateScript.Factory factory = engine.compile(null, template, TemplateScript.CONTEXT, Collections.emptyMap());
+        Map<String, Object> vars = new HashMap<>();
+        vars.put("data", new Object[] { singletonMap("key", "foo"), singletonMap("key", "bar") });
+        // assertThat(factory.newInstance(vars).execute(), equalTo("foo "));
+
+        vars.put("data", Arrays.asList(singletonMap("key", "foo"), singletonMap("key", "bar")));
+        factory.newInstance(vars);
+        assertThat(factory.newInstance(vars).execute(), equalTo("foo "));
+
+        // HashSet iteration order isn't fixed
+        Set<Object> setData = new HashSet<>();
+        setData.add(singletonMap("key", "foo"));
+        setData.add(singletonMap("key", "bar"));
+        vars.put("data", setData);
+        String output = factory.newInstance(vars).execute();
+        assertThat(output, both(containsString("foo")).and(not(containsString("bar"))));
+    }
+
     public void testSizeAccessForCollectionsAndArrays() throws Exception {
         String[] randomArrayValues = generateRandomStringArray(10, 20, false);
         List<String> randomList = Arrays.asList(generateRandomStringArray(10, 20, false));

muted-tests.yml

@@ -423,6 +423,18 @@ tests:
 - class: org.elasticsearch.smoketest.MlWithSecurityIT
   method: test {yaml=ml/get_trained_model_stats/Test get stats given trained models}
   issue: https://github.com/elastic/elasticsearch/issues/126510
+- class: org.elasticsearch.xpack.test.rest.XPackRestIT
+  method: test {p0=transform/transforms_stats/Test get multiple transform stats}
+  issue: https://github.com/elastic/elasticsearch/issues/126567
+- class: org.elasticsearch.xpack.test.rest.XPackRestIT
+  method: test {p0=transform/transforms_stats/Test get single transform stats when it does not have a task}
+  issue: https://github.com/elastic/elasticsearch/issues/126568
+- class: org.elasticsearch.repositories.blobstore.testkit.rest.SnapshotRepoTestKitClientYamlTestSuiteIT
+  method: test {p0=/10_analyze/Analysis without details}
+  issue: https://github.com/elastic/elasticsearch/issues/126569
+- class: org.elasticsearch.repositories.blobstore.testkit.analyze.S3RepositoryAnalysisRestIT
+  method: testRepositoryAnalysis
+  issue: https://github.com/elastic/elasticsearch/issues/126576
 
 # Examples:
 #

server/src/internalClusterTest/java/org/elasticsearch/index/shard/IndexShardIT.java

@@ -676,7 +676,7 @@ public void testInvalidateIndicesRequestCacheWhenRollbackEngine() throws Excepti
         final CountDownLatch engineResetLatch = new CountDownLatch(1);
         shard.acquireAllPrimaryOperationsPermits(ActionListener.wrap(r -> {
             try {
-                shard.resetEngineToGlobalCheckpoint();
+                shard.rollbackEngineToGlobalCheckpoint();
             } finally {
                 r.close();
                 engineResetLatch.countDown();

server/src/internalClusterTest/java/org/elasticsearch/indices/IndexingMemoryControllerIT.java

@@ -84,7 +84,8 @@ EngineConfig engineConfigWithLargerIndexingMemory(EngineConfig config) {
                 config.getRelativeTimeInNanosSupplier(),
                 config.getIndexCommitListener(),
                 config.isPromotableToPrimary(),
-                config.getMapperService()
+                config.getMapperService(),
+                config.getEngineResetLock()
             );
         }
 

server/src/internalClusterTest/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java

@@ -748,7 +748,7 @@ public void testDeletionOfFailingToRecoverIndexShouldStopRestore() throws Except
 
         logger.info("--> wait for the index to appear");
         // that would mean that recovery process started and failing
-        safeGet(clusterAdmin().prepareHealth(SAFE_AWAIT_TIMEOUT, "test-idx").execute());
+        awaitIndexExists("test-idx");
 
         logger.info("--> delete index");
         cluster().wipeIndices("test-idx");

server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java

@@ -150,11 +150,11 @@ static TopDocs mergeTopDocs(List<TopDocs> results, int topN, int from) {
             return topDocs;
         } else if (topDocs instanceof TopFieldGroups firstTopDocs) {
             final Sort sort = new Sort(firstTopDocs.fields);
-            final TopFieldGroups[] shardTopDocs = results.toArray(new TopFieldGroups[0]);
+            final TopFieldGroups[] shardTopDocs = results.stream().filter(td -> td != Lucene.EMPTY_TOP_DOCS).toArray(TopFieldGroups[]::new);
             mergedTopDocs = TopFieldGroups.merge(sort, from, topN, shardTopDocs, false);
         } else if (topDocs instanceof TopFieldDocs firstTopDocs) {
             final Sort sort = checkSameSortTypes(results, firstTopDocs.fields);
-            final TopFieldDocs[] shardTopDocs = results.toArray(new TopFieldDocs[0]);
+            final TopFieldDocs[] shardTopDocs = results.stream().filter((td -> td != Lucene.EMPTY_TOP_DOCS)).toArray(TopFieldDocs[]::new);
             mergedTopDocs = TopDocs.merge(sort, from, topN, shardTopDocs);
         } else {
             final TopDocs[] shardTopDocs = results.toArray(new TopDocs[numShards]);