Merge branch 'main' into index_micro_benchmarks_results

Author: idegtiarenko

build-tools-internal/src/main/resources/forbidden/es-all-signatures.txt

@@ -61,3 +61,11 @@ org.apache.logging.log4j.message.ParameterizedMessage#<init>(java.lang.String, j
 
 @defaultMessage Use WriteLoadForecaster#getForecastedWriteLoad instead
 org.elasticsearch.cluster.metadata.IndexMetadata#getForecastedWriteLoad()
+
+# This is a temporary patch as there is a low level Lucene bug in certain scenarios
+# this should be fixed in the new Lucene release 10.3+
+org.apache.lucene.document.LongField#newExactQuery(java.lang.String, long) @ Use org.elasticsearch.lucene.document.NumericField#newExactLongQuery(java.lang.String, long) instead.
+org.apache.lucene.document.LongField#newRangeQuery(java.lang.String, long, long) @ Use org.elasticsearch.lucene.document.NumericField#newRangeLongQuery(java.lang.String, long, long) instead.
+org.apache.lucene.document.IntField#newExactQuery(java.lang.String, int) @ Use org.elasticsearch.lucene.document.NumericField#newExactIntQuery(java.lang.String, int) instead.
+org.apache.lucene.document.IntField#newRangeQuery(java.lang.String, int, int) @ Use org.elasticsearch.lucene.document.NumericField#newRangeIntQuery(java.lang.String, int, int) instead.
+org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery @ use org.elasticsearch.lucene.search.XIndexSortSortedNumericDocValuesRangeQuery instead.

build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.java

@@ -20,10 +20,14 @@
 
 import static org.elasticsearch.gradle.internal.release.GenerateReleaseNotesTask.getSortedBundlesWithUniqueChangelogs;
 import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertThat;
 
 public class ReleaseNotesGeneratorTest {
 
+    // Temporarily set this to `true` to regenerate test output files when they need to be updated
+    private final boolean UPDATE_EXPECTED_OUTPUT = false;
+
     private static final List<String> CHANGE_TYPES = List.of(
         "breaking",
         "breaking-java",
@@ -78,8 +82,9 @@ public void testTemplate(String templateFilename, String outputFilename) throws
 
     public void testTemplate(String templateFilename, String outputFilename, List<ChangelogBundle> bundles) throws Exception {
         // given:
+        final String outputFile = "/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest." + outputFilename;
         final String template = getResource("/templates/" + templateFilename);
-        final String expectedOutput = getResource("/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest." + outputFilename);
+        final String expectedOutput = getResource(outputFile);
 
         if (bundles == null) {
             bundles = getBundles();
@@ -91,7 +96,12 @@ public void testTemplate(String templateFilename, String outputFilename, List<Ch
         final String actualOutput = ReleaseNotesGenerator.generateFile(template, bundles);
 
         // then:
-        assertThat(actualOutput, equalTo(expectedOutput));
+        if (UPDATE_EXPECTED_OUTPUT) {
+            writeResource(outputFile, actualOutput);
+            assertFalse("UPDATE_EXPECTED_OUTPUT should be set back to false after updating output", UPDATE_EXPECTED_OUTPUT);
+        } else {
+            assertThat(actualOutput, equalTo(expectedOutput));
+        }
     }
 
     private List<ChangelogBundle> getBundles() {
@@ -176,4 +186,9 @@ private ChangelogEntry makeHighlightsEntry(int pr, boolean notable) {
     private String getResource(String name) throws Exception {
         return Files.readString(Paths.get(Objects.requireNonNull(this.getClass().getResource(name)).toURI()), StandardCharsets.UTF_8);
     }
+
+    private void writeResource(String name, String contents) throws Exception {
+        String path = "src/test/resources" + name;
+        Files.writeString(Paths.get(path), contents);
+    }
 }

build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.md

@@ -20,10 +20,10 @@ To check for security updates, go to [Security announcements for the Elastic sta
 % ### Fixes [elasticsearch-next-fixes]
 % *
 
+## 9.1.0 [elasticsearch-9.1.0-release-notes]
 ```{applies_to}
 stack: coming 9.1.0
 ```
-## 9.1.0 [elasticsearch-9.1.0-release-notes]
 
 ### Highlights [elasticsearch-9.1.0-highlights]
 
@@ -47,10 +47,10 @@ Search:
 *  [#52](https://github.com/elastic/elasticsearch/pull/52)
 
 
+## 9.0.10 [elasticsearch-9.0.10-release-notes]
 ```{applies_to}
 stack: coming 9.0.10
 ```
-## 9.0.10 [elasticsearch-9.0.10-release-notes]
 
 ### Features and enhancements [elasticsearch-9.0.10-features-enhancements]
 

build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseNotesGeneratorTest.index.no-highlights.md

@@ -20,10 +20,10 @@ To check for security updates, go to [Security announcements for the Elastic sta
 % ### Fixes [elasticsearch-next-fixes]
 % *
 
+## 9.0.10 [elasticsearch-9.0.10-release-notes]
 ```{applies_to}
 stack: coming 9.0.10
 ```
-## 9.0.10 [elasticsearch-9.0.10-release-notes]
 
 ### Features and enhancements [elasticsearch-9.0.10-features-enhancements]
 

docs/changelog/127968.yaml

@@ -0,0 +1,6 @@
+pr: 127968
+summary: "Enable sort optimization on int, short and byte fields"
+area: Search
+type: enhancement
+issues:
+ - 127965

docs/changelog/128036.yaml

@@ -0,0 +1,6 @@
+pr: 128036
+summary: Fix inner hits + aggregations concurrency bug
+area: Search
+type: bug
+issues:
+ - 122419

docs/changelog/128176.yaml

@@ -0,0 +1,5 @@
+pr: 128176
+summary: Implement SAML custom attributes support for Identity Provider
+area: Authentication
+type: enhancement
+issues: []

docs/changelog/128740.yaml

@@ -0,0 +1,5 @@
+pr: 128740
+summary: Optimize sparse vector stats collection
+area: Stats
+type: enhancement
+issues: []

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java

@@ -376,14 +376,21 @@ boolean isTriviallyAllowed(Class<?> requestingClass) {
             generalLogger.debug("Entitlement trivially allowed from outermost frame");
             return true;
         }
-        if (SYSTEM_LAYER_MODULES.contains(requestingClass.getModule())) {
+        if (isTrustedSystemClass(requestingClass)) {
             generalLogger.debug("Entitlement trivially allowed from system module [{}]", requestingClass.getModule().getName());
             return true;
         }
         generalLogger.trace("Entitlement not trivially allowed");
         return false;
     }
 
+    /**
+     * The main decision point for what counts as a trusted built-in JDK class.
+     */
+    protected boolean isTrustedSystemClass(Class<?> requestingClass) {
+        return SYSTEM_LAYER_MODULES.contains(requestingClass.getModule());
+    }
+
     @Override
     public String toString() {
         return "PolicyManager{" + "serverEntitlements=" + serverEntitlements + ", pluginsEntitlements=" + pluginsEntitlements + '}';

modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/InnerHitsIT.java

@@ -26,6 +26,8 @@
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.metrics.TopHits;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
 import org.elasticsearch.search.sort.FieldSortBuilder;
@@ -51,6 +53,7 @@
 import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO;
 import static org.elasticsearch.join.query.JoinQueryBuilders.hasChildQuery;
 import static org.elasticsearch.join.query.JoinQueryBuilders.hasParentQuery;
+import static org.elasticsearch.search.aggregations.AggregationBuilders.topHits;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCountAndNoFailures;
@@ -64,6 +67,7 @@
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -698,4 +702,68 @@ public void testTooHighResultWindow() {
             )
         );
     }
+
+    public void testTopHitsOnParentChild() throws Exception {
+        assertAcked(
+            prepareCreate("idx").setMapping(
+                jsonBuilder().startObject()
+                    .startObject("_doc")
+                    .startObject("properties")
+                    .startObject("id")
+                    .field("type", "keyword")
+                    .endObject()
+                    .startObject("join_field")
+                    .field("type", "join")
+                    .startObject("relations")
+                    .field("parent", new String[] { "child1", "child2" })
+                    .endObject()
+                    .endObject()
+                    .endObject()
+                    .endObject()
+                    .endObject()
+            )
+        );
+        ensureGreen("idx");
+
+        List<IndexRequestBuilder> requestBuilders = new ArrayList<>();
+        int numDocs = scaledRandomIntBetween(10, 100);
+        int child1 = 0;
+        int child2 = 0;
+        int[] child1InnerObjects = new int[numDocs];
+        int[] child2InnerObjects = new int[numDocs];
+        for (int parent = 0; parent < numDocs; parent++) {
+            String parentId = String.format(Locale.ENGLISH, "p_%03d", parent);
+            requestBuilders.add(createIndexRequest("idx", "parent", parentId, null));
+
+            int numChildDocs = child1InnerObjects[parent] = scaledRandomIntBetween(1, numDocs);
+            int limit = child1 + numChildDocs;
+            for (; child1 < limit; child1++) {
+                requestBuilders.add(createIndexRequest("idx", "child1", String.format(Locale.ENGLISH, "c1_%04d", child1), parentId));
+            }
+            numChildDocs = child2InnerObjects[parent] = scaledRandomIntBetween(1, numDocs);
+            limit = child2 + numChildDocs;
+            for (; child2 < limit; child2++) {
+                requestBuilders.add(createIndexRequest("idx", "child2", String.format(Locale.ENGLISH, "c2_%04d", child2), parentId));
+            }
+        }
+
+        indexRandom(true, requestBuilders);
+        ensureSearchable();
+
+        QueryBuilder hasChildQuery = hasChildQuery("child2", matchAllQuery(), ScoreMode.None).innerHit(new InnerHitBuilder().setSize(2));
+        AggregationBuilder topHitsAgg = topHits("top-children").size(3);
+
+        assertNoFailuresAndResponse(prepareSearch("idx").setQuery(hasChildQuery).addAggregation(topHitsAgg), response -> {
+            assertHitCount(response, numDocs);
+
+            TopHits topHits = response.getAggregations().get("top-children");
+            SearchHits hits = topHits.getHits();
+            assertThat(hits.getHits().length, equalTo(3));
+
+            for (SearchHit hit : hits) {
+                SearchHits innerHits = hit.getInnerHits().get("child2");
+                assertThat(innerHits.getHits().length, lessThanOrEqualTo(2));
+            }
+        });
+    }
 }