Merge branch 'main' into geogrid_tech_preview

Author: craigtaverner

docs/changelog/133954.yaml

@@ -0,0 +1,6 @@
+pr: 133954
+summary: "ILM: Force merge on zero-replica cloned index before snapshotting for searchable snapshots"
+area: ILM+SLM
+type: enhancement
+issues:
+ - 75478

docs/changelog/134033.yaml

@@ -0,0 +1,6 @@
+pr: 134033
+summary: Fix FORK with union-types
+area: ES|QL
+type: bug
+issues:
+ - 133973

docs/docset.yml

@@ -35,7 +35,6 @@ subs:
   api-prereq-title:   "Prerequisites"
   api-description-title:   "Description"
   api-path-parms-title:   "Path parameters"
-  api-query-parms-title:   "Query parameters"
   api-request-body-title:   "Request body"
   api-examples-title:   "Examples"
   ecloud:   "Elastic Cloud"
@@ -49,7 +48,6 @@ subs:
   serverless-full:   "Elastic Cloud Serverless"
   serverless-short:   "Serverless"
   es-serverless:   "Elasticsearch Serverless"
-  ess-utm-params:   "?page=docs&placement=docs-body"
   cloud-only:   "This feature is designed for indirect use by Elasticsearch Service, Elastic Cloud Enterprise, and Elastic Cloud on Kubernetes. Direct use is not supported."
   security-app:   "Elastic Security app"
   stack-monitor-app:   "Stack Monitoring"
@@ -87,13 +85,10 @@ subs:
   search-snaps:   "searchable snapshots"
   slm:   "snapshot lifecycle management"
   slm-init:   "SLM"
-  dfeed:   "datafeed"
   dfeeds:   "datafeeds"
   dfeeds-cap:   "Datafeeds"
   anomaly-detect:   "anomaly detection"
   anomaly-jobs:   "anomaly detection jobs"
-  dataframe:   "data frame"
-  dataframe-cap:   "Data frame"
   transform:   "transform"
   transforms:   "transforms"
   transforms-cap:   "Transforms"
@@ -113,4 +108,3 @@ subs:
   nlp:   "natural language processing"
   index-manage-app:   "Index Management"
   connectors-app:   "Connectors"
-  ingest-pipelines-app:   "Ingest Pipelines"

modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/270_set_processor.yml

@@ -6,7 +6,7 @@ teardown:
         ignore: 404
 
 ---
-"Test set processor with template value":
+"Test set processor with a templated value":
   - do:
       ingest.put_pipeline:
         id: "1"
@@ -53,6 +53,7 @@ teardown:
         index: test
         id: "2"
   - match: { _source.foo: "hello" }
+
 ---
 "Test set processor with index change and require_alias":
   - do:
@@ -295,3 +296,38 @@ teardown:
   - match: { _source.result_2: "{{bar}}" }
   - match: { _source.old_bar: 2 }
   - match: { _source.bar: 3 }
+
+---
+"Test set processor with ignore_empty_value and a literal empty value":
+  - do:
+      ingest.put_pipeline:
+        id: "1"
+        body:  >
+          {
+            "processors": [
+              {
+                "set" : {
+                  "description": "we don't forbid this, even though it's a little silly",
+                  "field" : "foo",
+                  "value" : "",
+                  "ignore_empty_value" : true
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        id: "1"
+        pipeline: "1"
+        body: {
+          foo: "hello"
+        }
+
+  - do:
+      get:
+        index: test
+        id: "1"
+  - match: { _source.foo: "" }

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -203,10 +203,6 @@ static TransportVersion def(int id) {
     public static final TransportVersion ESQL_FIXED_INDEX_LIKE_8_19 = def(8_841_0_64);
     public static final TransportVersion INITIAL_ELASTICSEARCH_8_19_4 = def(8_841_0_68);
     public static final TransportVersion V_9_0_0 = def(9_000_0_09);
-    public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_1 = def(9_000_0_10);
-    public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_2 = def(9_000_0_11);
-    public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_3 = def(9_000_0_12);
-    public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_4 = def(9_000_0_13);
     public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_7 = def(9_000_0_16);
     public static final TransportVersion COHERE_BIT_EMBEDDING_TYPE_SUPPORT_ADDED = def(9_001_0_00);
     public static final TransportVersion REMOVE_SNAPSHOT_FAILURES = def(9_002_0_00);

server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java

@@ -64,7 +64,7 @@
 public abstract class TransportAbstractBulkAction extends HandledTransportAction<BulkRequest, BulkResponse> {
     private static final Logger logger = LogManager.getLogger(TransportAbstractBulkAction.class);
 
-    public static final Set<String> STREAMS_ALLOWED_PARAMS = new HashSet<>(8) {
+    public static final Set<String> STREAMS_ALLOWED_PARAMS = new HashSet<>(9) {
         {
             add("error_trace");
             add("filter_path");
@@ -73,6 +73,7 @@ public abstract class TransportAbstractBulkAction extends HandledTransportAction
             add("op_type");
             add("pretty");
             add("refresh");
+            add("require_data_stream");
             add("timeout");
         }
     };

server/src/main/java/org/elasticsearch/cluster/metadata/LifecycleExecutionState.java

@@ -40,7 +40,8 @@ public record LifecycleExecutionState(
     String snapshotName,
     String shrinkIndexName,
     String snapshotIndexName,
-    String downsampleIndexName
+    String downsampleIndexName,
+    String forceMergeCloneIndexName
 ) {
 
     public static final String ILM_CUSTOM_METADATA_KEY = "ilm";
@@ -64,6 +65,7 @@ public record LifecycleExecutionState(
     private static final String SNAPSHOT_INDEX_NAME = "snapshot_index_name";
     private static final String SHRINK_INDEX_NAME = "shrink_index_name";
     private static final String DOWNSAMPLE_INDEX_NAME = "rollup_index_name";
+    private static final String FORCE_MERGE_CLONE_INDEX_NAME = "force_merge_clone_index_name";
 
     public static final LifecycleExecutionState EMPTY_STATE = LifecycleExecutionState.builder().build();
 
@@ -89,7 +91,8 @@ public static Builder builder(LifecycleExecutionState state) {
             .setShrinkIndexName(state.shrinkIndexName)
             .setSnapshotIndexName(state.snapshotIndexName)
             .setDownsampleIndexName(state.downsampleIndexName)
-            .setStepTime(state.stepTime);
+            .setStepTime(state.stepTime)
+            .setForceMergeCloneIndexName(state.forceMergeCloneIndexName);
     }
 
     public static LifecycleExecutionState fromCustomMetadata(Map<String, String> customData) {
@@ -202,6 +205,10 @@ public static LifecycleExecutionState fromCustomMetadata(Map<String, String> cus
         if (downsampleIndexName != null) {
             builder.setDownsampleIndexName(downsampleIndexName);
         }
+        String forceMergeCloneIndexName = customData.get(FORCE_MERGE_CLONE_INDEX_NAME);
+        if (forceMergeCloneIndexName != null) {
+            builder.setForceMergeCloneIndexName(forceMergeCloneIndexName);
+        }
         return builder.build();
     }
 
@@ -274,6 +281,9 @@ public Map<String, String> asMap() {
         if (downsampleIndexName != null) {
             result.put(DOWNSAMPLE_INDEX_NAME, downsampleIndexName);
         }
+        if (forceMergeCloneIndexName != null) {
+            result.put(FORCE_MERGE_CLONE_INDEX_NAME, forceMergeCloneIndexName);
+        }
         return Collections.unmodifiableMap(result);
     }
 
@@ -307,6 +317,7 @@ public static class Builder {
         private String shrinkIndexName;
         private String snapshotIndexName;
         private String downsampleIndexName;
+        private String forceMergeCloneIndexName;
 
         public Builder setPhase(String phase) {
             this.phase = phase;
@@ -398,6 +409,11 @@ public Builder setDownsampleIndexName(String downsampleIndexName) {
             return this;
         }
 
+        public Builder setForceMergeCloneIndexName(String forceMergeCloneIndexName) {
+            this.forceMergeCloneIndexName = forceMergeCloneIndexName;
+            return this;
+        }
+
         public LifecycleExecutionState build() {
             return new LifecycleExecutionState(
                 phase,
@@ -417,7 +433,8 @@ public LifecycleExecutionState build() {
                 snapshotName,
                 shrinkIndexName,
                 snapshotIndexName,
-                downsampleIndexName
+                downsampleIndexName,
+                forceMergeCloneIndexName
             );
         }
     }

server/src/main/resources/transport/definitions/unreferable/initial_elasticsearch_9_0_1.csv

@@ -0,0 +1 @@
+9000010

server/src/main/resources/transport/definitions/unreferable/initial_elasticsearch_9_0_2.csv

@@ -0,0 +1 @@
+9000011

server/src/main/resources/transport/definitions/unreferable/initial_elasticsearch_9_0_3.csv

@@ -0,0 +1 @@
+9000012