Merge branch 'main' into tests/knn-fix-search-with-no-dimensions

Author: carlosdelest

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java renamed to benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.benchmark.compute.operator;
+package org.elasticsearch.benchmark._nightly.esql;
 
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.NumericDocValuesField;
@@ -85,10 +85,18 @@
 @State(Scope.Thread)
 @Fork(1)
 public class ValuesSourceReaderBenchmark {
+    private static final String[] SUPPORTED_LAYOUTS = new String[] { "in_order", "shuffled", "shuffled_singles" };
+    private static final String[] SUPPORTED_NAMES = new String[] {
+        "long",
+        "int",
+        "double",
+        "keyword",
+        "stored_keyword",
+        "3_stored_keywords" };
+
     private static final int BLOCK_LENGTH = 16 * 1024;
     private static final int INDEX_SIZE = 10 * BLOCK_LENGTH;
     private static final int COMMIT_INTERVAL = 500;
-    private static final BigArrays BIG_ARRAYS = BigArrays.NON_RECYCLING_INSTANCE;
     private static final BlockFactory blockFactory = BlockFactory.getInstance(
         new NoopCircuitBreaker("noop"),
         BigArrays.NON_RECYCLING_INSTANCE
@@ -104,8 +112,8 @@ static void selfTest() {
             ValuesSourceReaderBenchmark benchmark = new ValuesSourceReaderBenchmark();
             benchmark.setupIndex();
             try {
-                for (String layout : ValuesSourceReaderBenchmark.class.getField("layout").getAnnotationsByType(Param.class)[0].value()) {
-                    for (String name : ValuesSourceReaderBenchmark.class.getField("name").getAnnotationsByType(Param.class)[0].value()) {
+                for (String layout : ValuesSourceReaderBenchmark.SUPPORTED_LAYOUTS) {
+                    for (String name : ValuesSourceReaderBenchmark.SUPPORTED_NAMES) {
                         benchmark.layout = layout;
                         benchmark.name = name;
                         try {
@@ -119,7 +127,7 @@ static void selfTest() {
             } finally {
                 benchmark.teardownIndex();
             }
-        } catch (IOException | NoSuchFieldException e) {
+        } catch (IOException e) {
             throw new AssertionError(e);
         }
     }
@@ -321,10 +329,10 @@ public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() {
      *     each page has a single document rather than {@code BLOCK_SIZE} docs.</li>
      * </ul>
      */
-    @Param({ "in_order", "shuffled", "shuffled_singles" })
+    @Param({ "in_order", "shuffled" })
     public String layout;
 
-    @Param({ "long", "int", "double", "keyword", "stored_keyword", "3_stored_keywords" })
+    @Param({ "long", "keyword", "stored_keyword" })
     public String name;
 
     private Directory directory;

benchmarks/src/test/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmarkTests.java renamed to benchmarks/src/test/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmarkTests.java

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.benchmark.compute.operator;
+package org.elasticsearch.benchmark._nightly.esql;
 
 import org.elasticsearch.test.ESTestCase;
 

docs/changelog/130427.yaml

@@ -1,5 +1,17 @@
 pr: 130427
-summary: Disallow brackets in unquoted index pattersn
+summary: Disallow brackets in unquoted index patterns
 area: ES|QL
-type: bug
-issues: []
+type: breaking
+issues:
+  - 130378
+breaking:
+  title: Unquoted index patterns do not allow `(` and `)` characters
+  area: ES|QL
+  details: >-
+    Previously, ES|QL accepted unquoted index patterns containing brackets, such as `FROM index(1) | ENRICH policy(2)`.
+
+    This query syntax is no longer valid because it could conflict with subquery syntax, where brackets are used as delimiters.
+
+    Brackets are now only allowed in quoted index patterns. For example: `FROM "index(1)" | ENRICH "policy(2)"`.
+  impact: "This affects existing queries containing brackets in index or policy names, i.e. in FROM, ENRICH, and LOOKUP JOIN commands."
+  notable: false

docs/changelog/130939.yaml

@@ -0,0 +1,5 @@
+pr: 130939
+summary: Expose HTTP connection metrics to telemetry
+area: Network
+type: enhancement
+issues: []

docs/changelog/131032.yaml

@@ -0,0 +1,5 @@
+pr: 131032
+summary: "Fix: `GET _synonyms` returns synonyms with empty rules"
+area: Relevance
+type: bug
+issues: []

docs/reference/query-languages/esql/limitations.md

@@ -250,3 +250,6 @@ Work around this limitation by converting the field to single value with one of
 * CSV export from Discover shows no more than 10,000 rows. This limit only applies to the number of rows that are retrieved by the query and displayed in Discover. Queries and aggregations run on the full data set.
 * Querying many indices at once without any filters can cause an error in kibana which looks like `[esql] > Unexpected error from Elasticsearch: The content length (536885793) is bigger than the maximum allowed string (536870888)`. The response from {{esql}} is too long. Use [`DROP`](/reference/query-languages/esql/commands/processing-commands.md#esql-drop) or [`KEEP`](/reference/query-languages/esql/commands/processing-commands.md#esql-keep) to limit the number of fields returned.
 
+## Known issues [esql-known-issues]
+
+Refer to [Known issues](/release-notes/known-issues.md) for a list of known issues for {{esql}}.

docs/release-notes/known-issues.md

@@ -8,10 +8,19 @@ mapped_pages:
 Known issues are significant defects or limitations that may impact your implementation. These issues are actively being worked on and will be addressed in a future release. Review the Elasticsearch known issues to help you make informed decisions, such as upgrading to a new version.
 
 ## 9.0.3 [elasticsearch-9.0.3-known-issues]
-A bug in the merge scheduler in Elasticsearch 9.0.3 may prevent shards from closing when there isn’t enough disk space to complete a merge. As a result, operations such as closing or relocating an index may hang until sufficient disk space becomes available.
+* A bug in the merge scheduler in Elasticsearch 9.0.3 may prevent shards from closing when there isn’t enough disk space to complete a merge. As a result, operations such as closing or relocating an index may hang until sufficient disk space becomes available.
 To mitigate this issue, the disk space checker is disabled by default in 9.0.3 by setting `indices.merge.disk.check_interval` to `0` seconds. Manually enabling this setting is not recommended.
 
-This issue is planned to be fixed in future patch release [#129613](https://github.com/elastic/elasticsearch/pull/129613)
+  This issue is planned to be fixed in future patch release [#129613](https://github.com/elastic/elasticsearch/pull/129613)
+
+* A bug in the ES|QL STATS command may yield incorrect results. The bug only happens in very specific cases that follow this pattern: `STATS ... BY keyword1, keyword2`, i.e. the command must have exactly two grouping fields, both keywords, where the first field has high cardinality (more than 65k distinct values).
+
+  The bug is described in detail in [this issue](https://github.com/elastic/elasticsearch/issues/130644).
+  The problem was introduced in 8.16.0 and [fixed](https://github.com/elastic/elasticsearch/pull/130705) in 8.17.9, 8.18.7, 9.0.4.
+
+  Possible workarounds include:
+  * switching the order of the grouping keys (eg. `STATS ... BY keyword2, keyword1`, if the `keyword2` has a lower cardinality)
+  * reducing the grouping key cardinality, by filtering out values before STATS
 
 ## 9.0.0 [elasticsearch-9.0.0-known-issues]
 * Elasticsearch on Windows might fail to start, or might forbid some file-related operations, when referencing paths with a case different from the one stored by the filesystem. Windows treats paths as case-insensitive, but the filesystem stores them with case. Entitlements, the new security system used by Elasticsearch, treat all paths as case-sensitive, and can therefore prevent access to a path that should be accessible.
@@ -40,3 +49,12 @@ This issue will be fixed in a future patch release (see [PR #126990](https://git
      DELETE _index_template/.watches
      POST /_watcher/_start
      ```
+
+* A bug in the ES|QL STATS command may yield incorrect results. The bug only happens in very specific cases that follow this pattern: `STATS ... BY keyword1, keyword2`, i.e. the command must have exactly two grouping fields, both keywords, where the first field has high cardinality (more than 65k distinct values).
+
+  The bug is described in detail in [this issue](https://github.com/elastic/elasticsearch/issues/130644).
+  The problem was introduced in 8.16.0 and [fixed](https://github.com/elastic/elasticsearch/pull/130705) in 8.17.9, 8.18.7, 9.0.4.
+
+  Possible workarounds include:
+    * switching the order of the grouping keys (eg. `STATS ... BY keyword2, keyword1`, if the `keyword2` has a lower cardinality)
+    * reducing the grouping key cardinality, by filtering out values before STATS

libs/x-content/src/main/java/org/elasticsearch/xcontent/AbstractObjectParser.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.xcontent;
 
 import org.elasticsearch.core.CheckedFunction;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.xcontent.ObjectParser.NamedObjectParser;
 import org.elasticsearch.xcontent.ObjectParser.ValueType;
 
@@ -230,11 +231,13 @@ public void declareDoubleOrNull(BiConsumer<Value, Double> consumer, double nullV
         );
     }
 
+    @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA) // https://github.com/elastic/elasticsearch/issues/130797
     public void declareLong(BiConsumer<Value, Long> consumer, ParseField field) {
         // Using a method reference here angers some compilers
         declareField(consumer, p -> p.longValue(), field, ValueType.LONG);
     }
 
+    @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA) // https://github.com/elastic/elasticsearch/issues/130797
     public void declareLongOrNull(BiConsumer<Value, Long> consumer, long nullValue, ParseField field) {
         // Using a method reference here angers some compilers
         declareField(
@@ -245,6 +248,7 @@ public void declareLongOrNull(BiConsumer<Value, Long> consumer, long nullValue,
         );
     }
 
+    @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA) // https://github.com/elastic/elasticsearch/issues/130797
     public void declareInt(BiConsumer<Value, Integer> consumer, ParseField field) {
         // Using a method reference here angers some compilers
         declareField(consumer, p -> p.intValue(), field, ValueType.INT);
@@ -253,6 +257,7 @@ public void declareInt(BiConsumer<Value, Integer> consumer, ParseField field) {
     /**
      * Declare an integer field that parses explicit {@code null}s in the json to a default value.
      */
+    @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA) // https://github.com/elastic/elasticsearch/issues/130797
     public void declareIntOrNull(BiConsumer<Value, Integer> consumer, int nullValue, ParseField field) {
         declareField(
             consumer,
@@ -320,10 +325,12 @@ public void declareFloatArray(BiConsumer<Value, List<Float>> consumer, ParseFiel
         declareFieldArray(consumer, (p, c) -> p.floatValue(), field, ValueType.FLOAT_ARRAY);
     }
 
+    @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA) // https://github.com/elastic/elasticsearch/issues/130797
     public void declareLongArray(BiConsumer<Value, List<Long>> consumer, ParseField field) {
         declareFieldArray(consumer, (p, c) -> p.longValue(), field, ValueType.LONG_ARRAY);
     }
 
+    @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA) // https://github.com/elastic/elasticsearch/issues/130797
     public void declareIntArray(BiConsumer<Value, List<Integer>> consumer, ParseField field) {
         declareFieldArray(consumer, (p, c) -> p.intValue(), field, ValueType.INT_ARRAY);
     }

modules/reindex/src/test/java/org/elasticsearch/reindex/ReindexBasicTests.java

@@ -23,6 +23,7 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.index.IndexSettings.SYNTHETIC_VECTORS;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
@@ -181,6 +182,7 @@ public void testReindexFromComplexDateMathIndexName() throws Exception {
     }
 
     public void testReindexIncludeVectors() throws Exception {
+        assumeTrue("This test requires synthetic vectors to be enabled", SYNTHETIC_VECTORS);
         var resp1 = prepareCreate("test").setSettings(
             Settings.builder().put(IndexSettings.INDEX_MAPPING_SOURCE_SYNTHETIC_VECTORS_SETTING.getKey(), true).build()
         ).setMapping("foo", "type=dense_vector,similarity=l2_norm", "bar", "type=sparse_vector").get();

modules/reindex/src/test/java/org/elasticsearch/reindex/UpdateByQueryBasicTests.java

@@ -24,6 +24,7 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.index.IndexSettings.SYNTHETIC_VECTORS;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
@@ -157,6 +158,7 @@ public void testMissingSources() {
     }
 
     public void testUpdateByQueryIncludeVectors() throws Exception {
+        assumeTrue("This test requires synthetic vectors to be enabled", SYNTHETIC_VECTORS);
         var resp1 = prepareCreate("test").setSettings(
             Settings.builder().put(IndexSettings.INDEX_MAPPING_SOURCE_SYNTHETIC_VECTORS_SETTING.getKey(), true).build()
         ).setMapping("foo", "type=dense_vector,similarity=l2_norm", "bar", "type=sparse_vector").get();