Merge remote-tracking branch 'origin/main' into esql_lookup_join_types_table

Author: craigtaverner

docs/changelog/128866.yaml

@@ -0,0 +1,6 @@
+pr: 128866
+summary: Add `age_in_millis` to ILM Explain Response
+area: ILM+SLM
+type: enhancement
+issues:
+ - 103659

docs/reference/query-languages/esql/README.md

@@ -105,16 +105,95 @@ To help differentiate between the static and generated content, the generated co
 % This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
 ```
 
+## Version differentiation in Docs V3
+
+> [!IMPORTANT]
+> Starting with 9.0, we no longer publish separate documentation branches for every minor release (`9.0`, `9.1`, `9.2`, etc.). 
+> This means there won't be a different page for `9.1`, `9.2`, and so on. Instead, all changes landing in subsequent minor releases **will appear on the same page**.
+
+Because we now publish just one docs set off of the `main` branch, we use the [`applies_to` metadata](https://elastic.github.io/docs-builder/syntax/applies/) to differentiate features and their availability across different versions. This is a [cumulative approach](https://elastic.github.io/docs-builder/contribute/#cumulative-docs): instead of creating separate pages for each product and release, we update a **single page** with product- and version-specific details over time.
+
+`applies_to` allows us to clearly communicate when features are introduced, when they transition from preview to GA, and which versions support specific functionality.
+
+This metadata accepts a lifecycle and an optional version.
+
+### Functions and operators
+
+Use the `@FunctionAppliesTo` annotation within the `@FunctionInfo` annotation on function and operator classes to specify the lifecycle and version for functions and operators.
+
+For example, to indicate that a function is in technical preview and applies to version 9.0.0, you would use:
+
+```java
+@FunctionInfo(
+    returnType = "boolean",
+    appliesTo = {
+        @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.0.0")
+    },
+    ...
+)
+```
+
+When a feature evolves from preview in `9.0` to GA in `9.2`, add a new entry alongside the existing preview entry and remove the `preview = true` boolean:
+
+```java
+@FunctionInfo(
+    returnType = "boolean",
+    preview = false, //  the preview boolean can be removed (or flipped to false) when the function becomes GA
+    appliesTo = {
+        @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.PREVIEW, version = "9.0.0"),
+        @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.GA, version = "9.2.0")
+    },
+    ...
+)
+```
+
+We updated [`DocsV3Support.java`](https://github.com/elastic/elasticsearch/blob/main/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/DocsV3Support.java) to generate the `applies_to` metadata correctly for functions and operators.
+
+### Inline `applies_to` metadata
+
+Use [inline annotations](https://elastic.github.io/docs-builder/syntax/applies/#inline-annotations) to specify `applies_to` metadata in descriptions, parameter lists, etc.
+
+For example, the second item in this list is in technical preview as of version 9.2:
+
+```markdown
+- Item 1
+- Item 2 {applies_to}`stack: preview 9.2.`
+```
+
+### Key rules
+
+1. **Use the `preview = true` boolean** for any tech preview feature - this is required for the Kibana inline docs
+   - **Remove `preview = true`** only when the feature becomes GA on serverless and is _definitely_ going GA in the next minor release
+2. **Never delete `appliesTo` entries** - only add new ones as features evolve from preview to GA
+3. **Use specific versions** (`9.0.0`, `9.1.0`) when known, or just `PREVIEW` without a version if timing is uncertain
+4. **Add `applies_to` to examples** where necessary
+
+> [!IMPORTANT]
+> We don't use `applies_to` in the legacy asciidoc system for 8.x and earlier versions.
+
+### Supported lifecycles
+
+- `PREVIEW` - Feature is in technical preview
+- `GA` - Feature is generally available  
+- `DEPRECATED` - Feature is deprecated and will be removed in a future release
+- `UNAVAILABLE` - Feature is not available in the current version, but may be available in future releases
+
+> [!NOTE] 
+> Unreleased version information is automatically sanitized in the docs build output. For example, say you specify `preview 9.3.0`: 
+> - Before `9.3.0` is released, the live documentation will display "Planned for a future release" instead of the specific version number. 
+>  - This will be updated automatically when the version is released.
+
 ## Tutorials
 
 ### Adding a new command
 
 When adding a new command, for example adding the `CHANGE_POINT` command, do the following:
 1. Create a new file in the `_snippets/commands/layout` directory with the name of the command, for example `change_point.md`.
-2. Add the content for the command to the file. See other files in this directory for examples.
-3. Add the command to the list in `_snippets/lists/processing-commands.md`.
-4. Add an include directive to the `commands/processing-commands.md` file to include the new command.
-5. Add tested examples to the `_snippets/commands/examples` directory. See below for details.
+2. Ensure to specify what versions the command applies to. See [Version differentiation in Docs V3](#version-differentiation-in-docs-v3) for details. [Example PR](https://github.com/elastic/elasticsearch/pull/130314/files#diff-0ab90b6202c5d9eeea75dc95a7cb71dc4d720230342718bff887816771a5a803R3-R6).
+3. Add the content for the command to the file. See other files in this directory for examples.
+4. Add the command to the list in `_snippets/lists/processing-commands.md`.
+5. Add an include directive to the `commands/processing-commands.md` file to include the new command.
+6. Add tested examples to the `_snippets/commands/examples` directory. See below for details.
 
 ### Adding examples to commands
 

docs/reference/query-languages/esql/kibana/definition/commands/change_point.json

@@ -327,8 +327,6 @@ setup:
         index: "my-data-stream-1"
         wait_for_status: green
 
-
-
   - do:
       indices.get_data_stream:
         name: my-data-stream-1
@@ -398,3 +396,96 @@ setup:
   - match: { .$idx0name.settings.index.number_of_shards: "1" }
   - match: { .$idx0name.settings.index.lifecycle.name: "my-policy" }
   - match: { .$idx0name.settings.index.lifecycle.prefer_ilm: "true" }
+
+---
+"Test null out settings component templates only":
+  - requires:
+      cluster_features: [ "logs_stream" ]
+      reason: requires setting 'logs_stream' to get or set data stream settings
+
+  - do:
+      cluster.put_component_template:
+        name: settings-template
+        body:
+          template:
+            settings:
+              lifecycle.name: my-policy
+
+  - do:
+      allowed_warnings:
+        - "index template [my-component-only-template] has index patterns [my-component-only-data-stream-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-component-only-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: my-component-only-template
+        body:
+          index_patterns: [ my-component-only-data-stream-* ]
+          data_stream: { }
+          composed_of:
+            - settings-template
+
+  - do:
+      indices.create_data_stream:
+        name: my-component-only-data-stream-1
+
+  - do:
+      cluster.health:
+        index: "my-component-only-data-stream-1"
+        wait_for_status: green
+
+  - do:
+      indices.get_data_stream:
+        name: my-component-only-data-stream-1
+  - match: { data_streams.0.name: my-component-only-data-stream-1 }
+  - match: { data_streams.0.settings: {} }
+  - match: { data_streams.0.effective_settings: null }
+
+  - do:
+      indices.put_data_stream_settings:
+        name: my-component-only-data-stream-1
+        body:
+          index:
+            lifecycle:
+              name: my-new-policy
+              prefer_ilm: true
+  - match: { data_streams.0.name: my-component-only-data-stream-1 }
+  - match: { data_streams.0.applied_to_data_stream: true }
+  - length: { data_streams.0.index_settings_results.applied_to_data_stream_and_backing_indices: 2  }
+  - match: { data_streams.0.settings.index.lifecycle.name: "my-new-policy" }
+  - match: { data_streams.0.settings.index.lifecycle.prefer_ilm: "true" }
+  - match: { data_streams.0.effective_settings.index.lifecycle.name: "my-new-policy" }
+  - match: { data_streams.0.effective_settings.index.lifecycle.prefer_ilm: "true" }
+
+  - do:
+      indices.put_data_stream_settings:
+        name: my-component-only-data-stream-1
+        body:
+          index:
+            lifecycle:
+              name: null
+              prefer_ilm: null
+  - match: { data_streams.0.name: my-component-only-data-stream-1 }
+  - match: { data_streams.0.applied_to_data_stream: true }
+  - length: { data_streams.0.index_settings_results.applied_to_data_stream_and_backing_indices: 2  }
+  - match: { data_streams.0.settings.index.lifecycle.name: null }
+  - match: { data_streams.0.settings.index.lifecycle.prefer_ilm: null }
+  - match: { data_streams.0.effective_settings.index.lifecycle.name: "my-policy" }
+  - match: { data_streams.0.effective_settings.index.lifecycle.prefer_ilm: null }
+
+  - do:
+      indices.get_data_stream_settings:
+        name: my-component-only-data-stream-1
+  - match: { data_streams.0.name: my-component-only-data-stream-1 }
+  - match: { data_streams.0.settings.index.lifecycle.name: null }
+  - match: { data_streams.0.settings.index.lifecycle.prefer_ilm: null }
+  - match: { data_streams.0.effective_settings.index.lifecycle.name: "my-policy" }
+  - match: { data_streams.0.effective_settings.index.lifecycle.prefer_ilm: null }
+
+  - do:
+      indices.get_data_stream:
+        name: my-component-only-data-stream-1
+  - set: { data_streams.0.indices.0.index_name: idx0name }
+
+  - do:
+      indices.get_settings:
+        index: my-component-only-data-stream-1
+  - match: { .$idx0name.settings.index.lifecycle.name: "my-policy" }
+  - match: { .$idx0name.settings.index.lifecycle.prefer_ilm: null }

docs/reference/query-languages/esql/kibana/definition/functions/categorize.json

@@ -35,6 +35,7 @@
 import org.elasticsearch.common.text.UTF8DecodingReader;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.FieldDataContext;
@@ -105,8 +106,15 @@ public static class Builder extends FieldMapper.Builder {
 
         private final TextParams.Analyzers analyzers;
         private final boolean withinMultiField;
+        private final boolean storedFieldInBinaryFormat;
 
-        public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers indexAnalyzers, boolean withinMultiField) {
+        public Builder(
+            String name,
+            IndexVersion indexCreatedVersion,
+            IndexAnalyzers indexAnalyzers,
+            boolean withinMultiField,
+            boolean storedFieldInBinaryFormat
+        ) {
             super(name);
             this.indexCreatedVersion = indexCreatedVersion;
             this.analyzers = new TextParams.Analyzers(
@@ -116,6 +124,7 @@ public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers ind
                 indexCreatedVersion
             );
             this.withinMultiField = withinMultiField;
+            this.storedFieldInBinaryFormat = storedFieldInBinaryFormat;
         }
 
         @Override
@@ -135,7 +144,8 @@ private MatchOnlyTextFieldType buildFieldType(MapperBuilderContext context) {
                 context.isSourceSynthetic(),
                 meta.getValue(),
                 withinMultiField,
-                multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField()
+                multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField(),
+                storedFieldInBinaryFormat
             );
             return ft;
         }
@@ -155,8 +165,22 @@ public MatchOnlyTextFieldMapper build(MapperBuilderContext context) {
         }
     }
 
+    private static boolean isSyntheticSourceStoredFieldInBinaryFormat(IndexVersion indexCreatedVersion) {
+        return indexCreatedVersion.onOrAfter(IndexVersions.MATCH_ONLY_TEXT_STORED_AS_BYTES)
+            || indexCreatedVersion.between(
+                IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_BACKPORT_8_X,
+                IndexVersions.UPGRADE_TO_LUCENE_10_0_0
+            );
+    }
+
     public static final TypeParser PARSER = new TypeParser(
-        (n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers(), c.isWithinMultiField())
+        (n, c) -> new Builder(
+            n,
+            c.indexVersionCreated(),
+            c.getIndexAnalyzers(),
+            c.isWithinMultiField(),
+            isSyntheticSourceStoredFieldInBinaryFormat(c.indexVersionCreated())
+        )
     );
 
     public static class MatchOnlyTextFieldType extends StringFieldType {
@@ -167,6 +191,7 @@ public static class MatchOnlyTextFieldType extends StringFieldType {
 
         private final boolean withinMultiField;
         private final boolean hasCompatibleMultiFields;
+        private final boolean storedFieldInBinaryFormat;
 
         public MatchOnlyTextFieldType(
             String name,
@@ -175,14 +200,16 @@ public MatchOnlyTextFieldType(
             boolean isSyntheticSource,
             Map<String, String> meta,
             boolean withinMultiField,
-            boolean hasCompatibleMultiFields
+            boolean hasCompatibleMultiFields,
+            boolean storedFieldInBinaryFormat
         ) {
             super(name, true, false, false, tsi, meta);
             this.indexAnalyzer = Objects.requireNonNull(indexAnalyzer);
             this.textFieldType = new TextFieldType(name, isSyntheticSource);
             this.originalName = isSyntheticSource ? name + "._original" : null;
             this.withinMultiField = withinMultiField;
             this.hasCompatibleMultiFields = hasCompatibleMultiFields;
+            this.storedFieldInBinaryFormat = storedFieldInBinaryFormat;
         }
 
         public MatchOnlyTextFieldType(String name) {
@@ -193,6 +220,7 @@ public MatchOnlyTextFieldType(String name) {
                 false,
                 Collections.emptyMap(),
                 false,
+                false,
                 false
             );
         }
@@ -451,7 +479,11 @@ protected BytesRef toBytesRef(Object v) {
         @Override
         public BlockLoader blockLoader(BlockLoaderContext blContext) {
             if (textFieldType.isSyntheticSource()) {
-                return new BytesFromMixedStringsBytesRefBlockLoader(storedFieldNameForSyntheticSource());
+                if (storedFieldInBinaryFormat) {
+                    return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(storedFieldNameForSyntheticSource());
+                } else {
+                    return new BytesFromMixedStringsBytesRefBlockLoader(storedFieldNameForSyntheticSource());
+                }
             }
             SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name()));
             // MatchOnlyText never has norms, so we have to use the field names field
@@ -502,6 +534,7 @@ private String storedFieldNameForSyntheticSource() {
     private final boolean storeSource;
     private final FieldType fieldType;
     private final boolean withinMultiField;
+    private final boolean storedFieldInBinaryFormat;
 
     private MatchOnlyTextFieldMapper(
         String simpleName,
@@ -521,6 +554,7 @@ private MatchOnlyTextFieldMapper(
         this.positionIncrementGap = builder.analyzers.positionIncrementGap.getValue();
         this.storeSource = storeSource;
         this.withinMultiField = builder.withinMultiField;
+        this.storedFieldInBinaryFormat = builder.storedFieldInBinaryFormat;
     }
 
     @Override
@@ -530,7 +564,7 @@ public Map<String, NamedAnalyzer> indexAnalyzers() {
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(leafName(), indexCreatedVersion, indexAnalyzers, withinMultiField).init(this);
+        return new Builder(leafName(), indexCreatedVersion, indexAnalyzers, withinMultiField, storedFieldInBinaryFormat).init(this);
     }
 
     @Override
@@ -547,8 +581,12 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
         context.addToFieldNames(fieldType().name());
 
         if (storeSource) {
-            final var bytesRef = new BytesRef(utfBytes.bytes(), utfBytes.offset(), utfBytes.length());
-            context.doc().add(new StoredField(fieldType().storedFieldNameForSyntheticSource(), bytesRef));
+            if (storedFieldInBinaryFormat) {
+                final var bytesRef = new BytesRef(utfBytes.bytes(), utfBytes.offset(), utfBytes.length());
+                context.doc().add(new StoredField(fieldType().storedFieldNameForSyntheticSource(), bytesRef));
+            } else {
+                context.doc().add(new StoredField(fieldType().storedFieldNameForSyntheticSource(), value.string()));
+            }
         }
     }