Merge branch 'main' into pabloem-copy-sign

Author: pabloem

build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java

@@ -20,10 +20,14 @@
 
 import static org.elasticsearch.gradle.internal.release.GenerateReleaseNotesTask.getSortedBundlesWithUniqueChangelogs;
 import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertThat;
 
 public class ReleaseNotesGeneratorTest {
 
+    // Temporarily set this to `true` to regenerate test output files when they need to be updated
+    private final boolean UPDATE_EXPECTED_OUTPUT = false;
+
     private static final List<String> CHANGE_TYPES = List.of(
         "breaking",
         "breaking-java",
@@ -78,8 +82,9 @@ public void testTemplate(String templateFilename, String outputFilename) throws
 
     public void testTemplate(String templateFilename, String outputFilename, List<ChangelogBundle> bundles) throws Exception {
         // given:
+        final String outputFile = "/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest." + outputFilename;
         final String template = getResource("/templates/" + templateFilename);
-        final String expectedOutput = getResource("/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest." + outputFilename);
+        final String expectedOutput = getResource(outputFile);
 
         if (bundles == null) {
             bundles = getBundles();
@@ -91,7 +96,12 @@ public void testTemplate(String templateFilename, String outputFilename, List<Ch
         final String actualOutput = ReleaseNotesGenerator.generateFile(template, bundles);
 
         // then:
-        assertThat(actualOutput, equalTo(expectedOutput));
+        if (UPDATE_EXPECTED_OUTPUT) {
+            writeResource(outputFile, actualOutput);
+            assertFalse("UPDATE_EXPECTED_OUTPUT should be set back to false after updating output", UPDATE_EXPECTED_OUTPUT);
+        } else {
+            assertThat(actualOutput, equalTo(expectedOutput));
+        }
     }
 
     private List<ChangelogBundle> getBundles() {
@@ -176,4 +186,9 @@ private ChangelogEntry makeHighlightsEntry(int pr, boolean notable) {
     private String getResource(String name) throws Exception {
         return Files.readString(Paths.get(Objects.requireNonNull(this.getClass().getResource(name)).toURI()), StandardCharsets.UTF_8);
     }
+
+    private void writeResource(String name, String contents) throws Exception {
+        String path = "src/test/resources" + name;
+        Files.writeString(Paths.get(path), contents);
+    }
 }

build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.md

@@ -20,10 +20,10 @@ To check for security updates, go to [Security announcements for the Elastic sta
 % ### Fixes [elasticsearch-next-fixes]
 % *
 
+## 9.1.0 [elasticsearch-9.1.0-release-notes]
 ```{applies_to}
 stack: coming 9.1.0
 ```
-## 9.1.0 [elasticsearch-9.1.0-release-notes]
 
 ### Highlights [elasticsearch-9.1.0-highlights]
 
@@ -47,10 +47,10 @@ Search:
 *  [#52](https://github.com/elastic/elasticsearch/pull/52)
 
 
+## 9.0.10 [elasticsearch-9.0.10-release-notes]
 ```{applies_to}
 stack: coming 9.0.10
 ```
-## 9.0.10 [elasticsearch-9.0.10-release-notes]
 
 ### Features and enhancements [elasticsearch-9.0.10-features-enhancements]
 

build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.no-highlights.md

@@ -20,10 +20,10 @@ To check for security updates, go to [Security announcements for the Elastic sta
 % ### Fixes [elasticsearch-next-fixes]
 % *
 
+## 9.0.10 [elasticsearch-9.0.10-release-notes]
 ```{applies_to}
 stack: coming 9.0.10
 ```
-## 9.0.10 [elasticsearch-9.0.10-release-notes]
 
 ### Features and enhancements [elasticsearch-9.0.10-features-enhancements]
 

docs/changelog/127968.yaml

@@ -0,0 +1,6 @@
+pr: 127968
+summary: "Enable sort optimization on int, short and byte fields"
+area: Search
+type: enhancement
+issues:
+ - 127965

docs/changelog/128036.yaml

@@ -0,0 +1,6 @@
+pr: 128036
+summary: Fix inner hits + aggregations concurrency bug
+area: Search
+type: bug
+issues:
+ - 122419

modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/InnerHitsIT.java

@@ -26,6 +26,8 @@
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.TopHits;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
 import org.elasticsearch.search.sort.FieldSortBuilder;
@@ -51,6 +53,7 @@
 import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO;
 import static org.elasticsearch.join.query.JoinQueryBuilders.hasChildQuery;
 import static org.elasticsearch.join.query.JoinQueryBuilders.hasParentQuery;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.topHits;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCountAndNoFailures;
@@ -64,6 +67,7 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -698,4 +702,68 @@ public void testTooHighResultWindow() {
             )
         );
     }
+
+    public void testTopHitsOnParentChild() throws Exception {
+        assertAcked(
+            prepareCreate("idx").setMapping(
+                jsonBuilder().startObject()
+                    .startObject("_doc")
+                    .startObject("properties")
+                    .startObject("id")
+                    .field("type", "keyword")
+                    .endObject()
+                    .startObject("join_field")
+                    .field("type", "join")
+                    .startObject("relations")
+                    .field("parent", new String[] { "child1", "child2" })
+                    .endObject()
+                    .endObject()
+                    .endObject()
+                    .endObject()
+                    .endObject()
+            )
+        );
+        ensureGreen("idx");
+
+        List<IndexRequestBuilder> requestBuilders = new ArrayList<>();
+        int numDocs = scaledRandomIntBetween(10, 100);
+        int child1 = 0;
+        int child2 = 0;
+        int[] child1InnerObjects = new int[numDocs];
+        int[] child2InnerObjects = new int[numDocs];
+        for (int parent = 0; parent < numDocs; parent++) {
+            String parentId = String.format(Locale.ENGLISH, "p_%03d", parent);
+            requestBuilders.add(createIndexRequest("idx", "parent", parentId, null));
+
+            int numChildDocs = child1InnerObjects[parent] = scaledRandomIntBetween(1, numDocs);
+            int limit = child1 + numChildDocs;
+            for (; child1 < limit; child1++) {
+                requestBuilders.add(createIndexRequest("idx", "child1", String.format(Locale.ENGLISH, "c1_%04d", child1), parentId));
+            }
+            numChildDocs = child2InnerObjects[parent] = scaledRandomIntBetween(1, numDocs);
+            limit = child2 + numChildDocs;
+            for (; child2 < limit; child2++) {
+                requestBuilders.add(createIndexRequest("idx", "child2", String.format(Locale.ENGLISH, "c2_%04d", child2), parentId));
+            }
+        }
+
+        indexRandom(true, requestBuilders);
+        ensureSearchable();
+
+        QueryBuilder hasChildQuery = hasChildQuery("child2", matchAllQuery(), ScoreMode.None).innerHit(new InnerHitBuilder().setSize(2));
+        AggregationBuilder topHitsAgg = topHits("top-children").size(3);
+
+        assertNoFailuresAndResponse(prepareSearch("idx").setQuery(hasChildQuery).addAggregation(topHitsAgg), response -> {
+            assertHitCount(response, numDocs);
+
+            TopHits topHits = response.getAggregations().get("top-children");
+            SearchHits hits = topHits.getHits();
+            assertThat(hits.getHits().length, equalTo(3));
+
+            for (SearchHit hit : hits) {
+                SearchHits innerHits = hit.getInnerHits().get("child2");
+                assertThat(innerHits.getHits().length, lessThanOrEqualTo(2));
+            }
+        });
+    }
 }

modules/parent-join/src/main/java/org/elasticsearch/join/query/ParentChildInnerHitContextBuilder.java

@@ -88,12 +88,35 @@ static final class JoinFieldInnerHitSubContext extends InnerHitsContext.InnerHit
         private final String typeName;
         private final boolean fetchChildInnerHits;
         private final Joiner joiner;
+        private final SearchExecutionContext searchExecutionContext;
 
         JoinFieldInnerHitSubContext(String name, SearchContext context, String typeName, boolean fetchChildInnerHits, Joiner joiner) {
             super(name, context);
             this.typeName = typeName;
             this.fetchChildInnerHits = fetchChildInnerHits;
             this.joiner = joiner;
+            this.searchExecutionContext = null;
+        }
+
+        JoinFieldInnerHitSubContext(
+            JoinFieldInnerHitSubContext joinFieldInnerHitSubContext,
+            SearchExecutionContext searchExecutionContext
+        ) {
+            super(joinFieldInnerHitSubContext);
+            this.typeName = joinFieldInnerHitSubContext.typeName;
+            this.fetchChildInnerHits = joinFieldInnerHitSubContext.fetchChildInnerHits;
+            this.joiner = joinFieldInnerHitSubContext.joiner;
+            this.searchExecutionContext = searchExecutionContext;
+        }
+
+        @Override
+        public JoinFieldInnerHitSubContext copyWithSearchExecutionContext(SearchExecutionContext searchExecutionContext) {
+            return new JoinFieldInnerHitSubContext(this, searchExecutionContext);
+        }
+
+        @Override
+        public SearchExecutionContext getSearchExecutionContext() {
+            return searchExecutionContext != null ? searchExecutionContext : super.getSearchExecutionContext();
         }
 
         @Override

muted-tests.yml

@@ -534,12 +534,6 @@ tests:
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test040JavaUsesTheOsProvidedKeystore
   issue: https://github.com/elastic/elasticsearch/issues/128230
-- class: org.elasticsearch.gradle.internal.release.ReleaseNotesGeneratorTest
-  method: generateFile_index_rendersCorrectMarkup
-  issue: https://github.com/elastic/elasticsearch/issues/128772
-- class: org.elasticsearch.gradle.internal.release.ReleaseNotesGeneratorTest
-  method: generateFile_index_noHighlights_rendersCorrectMarkup
-  issue: https://github.com/elastic/elasticsearch/issues/128773
 - class: org.elasticsearch.xpack.rank.rrf.LinearRankClientYamlTestSuiteIT
   method: test {yaml=linear/10_linear_retriever/should normalize initial scores with l2_norm}
   issue: https://github.com/elastic/elasticsearch/issues/128774

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/IndexSortUpgradeIT.java

@@ -0,0 +1,100 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.upgrades;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.common.settings.Settings;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Tests that index sorting works correctly after a rolling upgrade.
+ */
+public class IndexSortUpgradeIT extends AbstractRollingUpgradeTestCase {
+
+    public IndexSortUpgradeIT(@Name("upgradedNodes") int upgradedNodes) {
+        super(upgradedNodes);
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testIndexSortForNumericTypes() throws Exception {
+        record IndexConfig(String indexName, String fieldName, String fieldType) {}
+        var configs = new IndexConfig[] {
+            new IndexConfig("index_byte", "byte_field", "byte"),
+            new IndexConfig("index_short", "short_field", "short"),
+            new IndexConfig("index_int", "int_field", "integer") };
+
+        if (isOldCluster()) {
+            int numShards = randomIntBetween(1, 3);
+            for (var config : configs) {
+                createIndex(
+                    config.indexName(),
+                    Settings.builder()
+                        .put("index.number_of_shards", numShards)
+                        .put("index.number_of_replicas", 0)
+                        .put("index.sort.field", config.fieldName())
+                        .put("index.sort.order", "desc")
+                        .build(),
+                    """
+                        {
+                            "properties": {
+                                "%s": {
+                                    "type": "%s"
+                                }
+                            }
+                        }
+                        """.formatted(config.fieldName(), config.fieldType())
+                );
+            }
+        }
+
+        final int numDocs = randomIntBetween(10, 25);
+        for (var config : configs) {
+            var bulkRequest = new Request("POST", "/" + config.indexName() + "/_bulk");
+            StringBuilder bulkBody = new StringBuilder();
+            for (int i = 0; i < numDocs; i++) {
+                bulkBody.append("{\"index\": {}}\n");
+                bulkBody.append("{\"" + config.fieldName() + "\": ").append(i).append("}\n");
+            }
+            bulkRequest.setJsonEntity(bulkBody.toString());
+            bulkRequest.addParameter("refresh", "true");
+            var bulkResponse = client().performRequest(bulkRequest);
+            assertOK(bulkResponse);
+
+            var searchRequest = new Request("GET", "/" + config.indexName() + "/_search");
+            searchRequest.setJsonEntity("""
+                {
+                    "query": {
+                        "match_all": {}
+                    },
+                    "sort": {
+                        "%s": {
+                            "order": "desc"
+                        }
+                    }
+                }
+                """.formatted(config.fieldName()));
+            var searchResponse = client().performRequest(searchRequest);
+            assertOK(searchResponse);
+            var responseBody = entityAsMap(searchResponse);
+            var hits = (List<Map<String, Object>>) ((Map<String, Object>) responseBody.get("hits")).get("hits");
+            int previousValue = ((Number) ((List<Object>) hits.get(0).get("sort")).get(0)).intValue();
+            ;
+            for (int i = 1; i < hits.size(); i++) {
+                int currentValue = ((Number) ((List<Object>) hits.get(i).get("sort")).get(0)).intValue();
+                assertTrue("Sort values are not in desc order ", previousValue >= currentValue);
+                previousValue = currentValue;
+            }
+        }
+    }
+}