Merge branch 'main' into fix/testApmIntegration-2

Author: ldematte

docs/release-notes/breaking-changes.md

@@ -12,23 +12,15 @@ If you are migrating from a version prior to version 9.0, you must first upgrade
 
 % ## Next version [elasticsearch-nextversion-breaking-changes]
 
-```{applies_to}
-stack: coming 9.0.3
-```
 ## 9.0.3 [elasticsearch-9.0.3-breaking-changes]
 
 No breaking changes in this version.
 
-```{applies_to}
-stack: coming 9.0.2
-```
 ## 9.0.2 [elasticsearch-9.0.2-breaking-changes]
 
 Snapshot/Restore:
 * Make S3 custom query parameter optional [#128043](https://github.com/elastic/elasticsearch/pull/128043)
 
-
-
 ## 9.0.1 [elasticsearch-9.0.1-breaking-changes]
 
 No breaking changes in this version.

docs/release-notes/changelog-bundles/9.0.2.yml

@@ -1,6 +1,6 @@
 version: 9.0.2
-released: false
-generated: 2025-05-22T15:14:00.768080Z
+released: true
+generated: 2025-06-03T14:58:36.937148609Z
 changelogs:
   - pr: 126992
     summary: Add missing `outbound_network` entitlement to x-pack-core
@@ -36,6 +36,12 @@ changelogs:
     type: bug
     issues:
       - 127468
+  - pr: 127564
+    summary: Consider inlinestats when having `field_caps` check for field names
+    area: ES|QL
+    type: bug
+    issues:
+      - 127236
   - pr: 127658
     summary: Append all data to Chat Completion buffer
     area: Machine Learning
@@ -133,3 +139,31 @@ changelogs:
     area: Data streams
     type: bug
     issues: []
+  - pr: 128259
+    summary: Added geometry validation for GEO types to exit early on invalid latitudes
+    area: Geo
+    type: bug
+    issues:
+      - 128234
+  - pr: 128260
+    summary: Fix validation NPE in Enrich and add extra @Nullable annotations
+    area: ES|QL
+    type: bug
+    issues:
+      - 126297
+      - 126253
+  - pr: 128320
+    summary: Use new source loader when lower `docId` is accessed
+    area: Codec
+    type: bug
+    issues: []
+  - pr: 128327
+    summary: Use internal user for internal inference action
+    area: Machine Learning
+    type: bug
+    issues: []
+  - pr: 128338
+    summary: Pass timeout to chat completion
+    area: Machine Learning
+    type: bug
+    issues: []

docs/release-notes/changelog-bundles/9.0.3.yml

@@ -1,6 +1,6 @@
 version: 9.0.3
-released: false
-generated: 2025-06-21T00:06:16.346021604Z
+released: true
+generated: 2025-06-24T15:19:29.859630035Z
 changelogs:
   - pr: 120869
     summary: Threadpool merge scheduler

docs/release-notes/deprecations.md

@@ -16,19 +16,11 @@ To give you insight into what deprecated features you’re using, {{es}}:
 
 % ## Next version [elasticsearch-nextversion-deprecations]
 
-```{applies_to}
-stack: coming 9.0.3
-```
 ## 9.0.3 [elasticsearch-9.0.3-deprecations]
 
 Engine:
 * Deprecate `indices.merge.scheduler.use_thread_pool` setting [#129464](https://github.com/elastic/elasticsearch/pull/129464)
 
-
-
-```{applies_to}
-stack: coming 9.0.2
-```
 ## 9.0.2 [elasticsearch-9.0.2-deprecations]
 
 No deprecations in this version.

docs/release-notes/index.md

@@ -21,9 +21,6 @@ To check for security updates, go to [Security announcements for the Elastic sta
 % *
 
 ## 9.0.3 [elasticsearch-9.0.3-release-notes]
-```{applies_to}
-stack: coming 9.0.3
-```
 
 ### Features and enhancements [elasticsearch-9.0.3-features-enhancements]
 
@@ -92,11 +89,7 @@ Searchable Snapshots:
 Security:
 * Fix error message when changing the password for a user in the file realm [#127621](https://github.com/elastic/elasticsearch/pull/127621)
 
-
 ## 9.0.2 [elasticsearch-9.0.2-release-notes]
-```{applies_to}
-stack: coming 9.0.2
-```
 
 ### Features and enhancements [elasticsearch-9.0.2-features-enhancements]
 
@@ -114,19 +107,27 @@ Aggregations:
 Audit:
 * Handle streaming request body in audit log [#127798](https://github.com/elastic/elasticsearch/pull/127798)
 
+Codec:
+* Use new source loader when lower `docId` is accessed [#128320](https://github.com/elastic/elasticsearch/pull/128320)
+
 Data streams:
 * Fix system data streams incorrectly showing up in the list of template validation problems [#128161](https://github.com/elastic/elasticsearch/pull/128161)
 
 Downsampling:
 * Downsampling does not consider passthrough fields as dimensions [#127752](https://github.com/elastic/elasticsearch/pull/127752) (issue: [#125156](https://github.com/elastic/elasticsearch/issues/125156))
 
 ES|QL:
+* Consider inlinestats when having `field_caps` check for field names [#127564](https://github.com/elastic/elasticsearch/pull/127564) (issue: [#127236](https://github.com/elastic/elasticsearch/issues/127236))
 * Don't push down filters on the right hand side of an inlinejoin [#127383](https://github.com/elastic/elasticsearch/pull/127383)
 * ESQL: Avoid unintended attribute removal [#127563](https://github.com/elastic/elasticsearch/pull/127563) (issue: [#127468](https://github.com/elastic/elasticsearch/issues/127468))
 * ESQL: Fix alias removal in regex extraction with JOIN [#127687](https://github.com/elastic/elasticsearch/pull/127687) (issue: [#127467](https://github.com/elastic/elasticsearch/issues/127467))
 * ESQL: Keep `DROP` attributes when resolving field names [#127009](https://github.com/elastic/elasticsearch/pull/127009) (issue: [#126418](https://github.com/elastic/elasticsearch/issues/126418))
 * Ensure ordinal builder emit ordinal blocks [#127949](https://github.com/elastic/elasticsearch/pull/127949)
 * Fix union types in CCS [#128111](https://github.com/elastic/elasticsearch/pull/128111)
+* Fix validation NPE in Enrich and add extra @Nullable annotations [#128260](https://github.com/elastic/elasticsearch/pull/128260) (issues: [#126297](https://github.com/elastic/elasticsearch/issues/126297), [#126253](https://github.com/elastic/elasticsearch/issues/126253))
+
+Geo:
+* Added geometry validation for GEO types to exit early on invalid latitudes [#128259](https://github.com/elastic/elasticsearch/pull/128259) (issue: [#128234](https://github.com/elastic/elasticsearch/issues/128234))
 
 Infra/Core:
 * Add missing `outbound_network` entitlement to x-pack-core [#126992](https://github.com/elastic/elasticsearch/pull/126992) (issue: [#127003](https://github.com/elastic/elasticsearch/issues/127003))
@@ -138,6 +139,8 @@ Infra/Scripting:
 Machine Learning:
 * Append all data to Chat Completion buffer [#127658](https://github.com/elastic/elasticsearch/pull/127658)
 * Fix services API Google Vertex AI Rerank location field requirement [#127856](https://github.com/elastic/elasticsearch/pull/127856)
+* Pass timeout to chat completion [#128338](https://github.com/elastic/elasticsearch/pull/128338)
+* Use internal user for internal inference action [#128327](https://github.com/elastic/elasticsearch/pull/128327)
 
 Relevance:
 * Fix: Add `NamedWriteable` for `RuleQueryRankDoc` [#128153](https://github.com/elastic/elasticsearch/pull/128153) (issue: [#126071](https://github.com/elastic/elasticsearch/issues/126071))

libs/core/src/main/java/org/elasticsearch/core/NotMultiProjectCapable.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.core;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Annotation to identify a block of code (a whole class, a method, a field, or a local variable) that is intentionally not fully
+ * project-aware because it's not intended to be used in a serverless environment. Some features are unavailable in serverless and are
+ * thus not worth the investment to make fully project-aware. This annotation makes it easier to identify blocks of code that require
+ * attention in case those features are revisited from a multi-project POV.
+ */
+@Retention(RetentionPolicy.SOURCE)
+@Target(
+    { ElementType.LOCAL_VARIABLE, ElementType.CONSTRUCTOR, ElementType.FIELD, ElementType.METHOD, ElementType.TYPE, ElementType.MODULE }
+)
+public @interface NotMultiProjectCapable {
+
+    /**
+     * Some explanation on why the block of code would not work in a multi-project context and/or what would need to be done to make it
+     * properly project-aware.
+     */
+    String description() default "";
+}

muted-tests.yml

@@ -561,7 +561,12 @@ tests:
   method: "builds distribution from branches via archives extractedAssemble [bwcDistVersion: 8.2.1, bwcProject: bugfix, expectedAssembleTaskName:
     extractedAssemble, #2]"
   issue: https://github.com/elastic/elasticsearch/issues/119871
-
+- class: org.elasticsearch.xpack.inference.qa.mixed.CohereServiceMixedIT
+  method: testRerank
+  issue: https://github.com/elastic/elasticsearch/issues/130009
+- class: org.elasticsearch.xpack.inference.qa.mixed.CohereServiceMixedIT
+  method: testCohereEmbeddings
+  issue: https://github.com/elastic/elasticsearch/issues/130010
 
 # Examples:
 #

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java

@@ -7,13 +7,17 @@
 
 package org.elasticsearch.xpack.esql.planner;
 
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.common.logging.HeaderWarning;
+import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.compute.aggregation.AggregatorMode;
 import org.elasticsearch.compute.aggregation.GroupingAggregator;
 import org.elasticsearch.compute.aggregation.blockhash.BlockHash;
@@ -76,7 +80,6 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.Function;
@@ -177,6 +180,13 @@ private BlockLoader getBlockLoaderFor(int shardId, Attribute attr, MappedFieldTy
 
     /** A hack to pretend an unmapped field still exists. */
     private static class DefaultShardContextForUnmappedField extends DefaultShardContext {
+        private static final FieldType UNMAPPED_FIELD_TYPE = new FieldType(KeywordFieldMapper.Defaults.FIELD_TYPE);
+        static {
+            UNMAPPED_FIELD_TYPE.setDocValuesType(DocValuesType.NONE);
+            UNMAPPED_FIELD_TYPE.setIndexOptions(IndexOptions.NONE);
+            UNMAPPED_FIELD_TYPE.setStored(false);
+            UNMAPPED_FIELD_TYPE.freeze();
+        }
         private final KeywordEsField unmappedEsField;
 
         DefaultShardContextForUnmappedField(DefaultShardContext ctx, PotentiallyUnmappedKeywordEsField unmappedEsField) {
@@ -187,9 +197,22 @@ private static class DefaultShardContextForUnmappedField extends DefaultShardCon
         @Override
         public @Nullable MappedFieldType fieldType(String name) {
             var superResult = super.fieldType(name);
-            return superResult == null && name.equals(unmappedEsField.getName())
-                ? new KeywordFieldMapper.KeywordFieldType(name, false /* isIndexed */, false /* hasDocValues */, Map.of() /* meta */)
-                : superResult;
+            return superResult == null && name.equals(unmappedEsField.getName()) ? createUnmappedFieldType(name, this) : superResult;
+        }
+
+        static MappedFieldType createUnmappedFieldType(String name, DefaultShardContext context) {
+            var builder = new KeywordFieldMapper.Builder(name, context.ctx.indexVersionCreated());
+            builder.docValues(false);
+            builder.indexed(false);
+            return new KeywordFieldMapper.KeywordFieldType(
+                name,
+                UNMAPPED_FIELD_TYPE,
+                Lucene.KEYWORD_ANALYZER,
+                Lucene.KEYWORD_ANALYZER,
+                Lucene.KEYWORD_ANALYZER,
+                builder,
+                context.ctx.isSourceSynthetic()
+            );
         }
     }
 

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlannerTests.java

@@ -23,13 +23,18 @@
 import org.elasticsearch.compute.lucene.DataPartitioning;
 import org.elasticsearch.compute.lucene.LuceneSourceOperator;
 import org.elasticsearch.compute.lucene.LuceneTopNSourceOperator;
+import org.elasticsearch.compute.lucene.ValuesSourceReaderOperator;
 import org.elasticsearch.compute.operator.SourceOperator;
 import org.elasticsearch.compute.test.TestBlockFactory;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.cache.query.TrivialQueryCachingPolicy;
+import org.elasticsearch.index.mapper.BlockLoader;
+import org.elasticsearch.index.mapper.BlockSourceReader;
+import org.elasticsearch.index.mapper.FallbackSyntheticSourceBlockLoader;
+import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperServiceTestCase;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.plugins.ExtensiblePlugin;
@@ -42,10 +47,12 @@
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
+import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
 import org.elasticsearch.xpack.esql.expression.Order;
 import org.elasticsearch.xpack.esql.index.EsIndex;
 import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec;
+import org.elasticsearch.xpack.esql.plan.physical.FieldExtractExec;
 import org.elasticsearch.xpack.esql.plan.physical.LimitExec;
 import org.elasticsearch.xpack.esql.plan.physical.ParallelExec;
 import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
@@ -64,6 +71,7 @@
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 
 public class LocalExecutionPlannerTests extends MapperServiceTestCase {
@@ -84,10 +92,17 @@ public static Iterable<Object[]> parameters() throws Exception {
 
     private final ArrayList<Releasable> releasables = new ArrayList<>();
 
+    private Settings settings = SETTINGS;
+
     public LocalExecutionPlannerTests(@Name("estimatedRowSizeIsHuge") boolean estimatedRowSizeIsHuge) {
         this.estimatedRowSizeIsHuge = estimatedRowSizeIsHuge;
     }
 
+    @Override
+    protected Settings getIndexSettings() {
+        return settings;
+    }
+
     @Override
     protected Collection<Plugin> getPlugins() {
         var plugin = new SpatialPlugin();
@@ -229,6 +244,47 @@ public void testParallel() throws Exception {
         assertThat(plan.driverFactories, hasSize(2));
     }
 
+    public void testPlanUnmappedFieldExtractStoredSource() throws Exception {
+        var blockLoader = constructBlockLoader();
+        // In case of stored source we expect bytes based block source loader (this loads source from _source)
+        assertThat(blockLoader, instanceOf(BlockSourceReader.BytesRefsBlockLoader.class));
+    }
+
+    public void testPlanUnmappedFieldExtractSyntheticSource() throws Exception {
+        // Enables synthetic source, so that fallback synthetic source blocker loader is used:
+        settings = Settings.builder().put(settings).put("index.mapping.source.mode", "synthetic").build();
+
+        var blockLoader = constructBlockLoader();
+        // In case of synthetic source we expect bytes based block source loader (this loads source from _ignored_source)
+        assertThat(blockLoader, instanceOf(FallbackSyntheticSourceBlockLoader.class));
+    }
+
+    private BlockLoader constructBlockLoader() throws IOException {
+        EsQueryExec queryExec = new EsQueryExec(
+            Source.EMPTY,
+            index().name(),
+            IndexMode.STANDARD,
+            index().indexNameWithModes(),
+            List.of(new FieldAttribute(Source.EMPTY, EsQueryExec.DOC_ID_FIELD.getName(), EsQueryExec.DOC_ID_FIELD)),
+            null,
+            null,
+            null,
+            between(1, 1000)
+        );
+        FieldExtractExec fieldExtractExec = new FieldExtractExec(
+            Source.EMPTY,
+            queryExec,
+            List.of(
+                new FieldAttribute(Source.EMPTY, "potentially_unmapped", new PotentiallyUnmappedKeywordEsField("potentially_unmapped"))
+            ),
+            MappedFieldType.FieldExtractPreference.NONE
+        );
+        LocalExecutionPlanner.LocalExecutionPlan plan = planner().plan("test", FoldContext.small(), fieldExtractExec);
+        var p = plan.driverFactories.get(0).driverSupplier().physicalOperation();
+        var fieldInfo = ((ValuesSourceReaderOperator.Factory) p.intermediateOperatorFactories.get(0)).fields().get(0);
+        return fieldInfo.blockLoader().apply(0);
+    }
+
     private int randomEstimatedRowSize(boolean huge) {
         int hugeBoundary = SourceOperator.MIN_TARGET_PAGE_SIZE * 10;
         return huge ? between(hugeBoundary, Integer.MAX_VALUE) : between(1, hugeBoundary);