Merge branch 'main' into StatementParserTests

Author: fang-xing-esql

catalog-info.yaml

@@ -123,7 +123,7 @@ spec:
       pipeline_file: .buildkite/pipelines/lucene-snapshot/build-snapshot.yml
       env:
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
-        SLACK_NOTIFICATIONS_CHANNEL: "#lucene"
+        SLACK_NOTIFICATIONS_CHANNEL: "#lucene-ci"
         SLACK_NOTIFICATIONS_ALL_BRANCHES: "true"
       branch_configuration: lucene_snapshot
       default_branch: lucene_snapshot
@@ -167,7 +167,7 @@ spec:
       pipeline_file: .buildkite/pipelines/lucene-snapshot/update-branch.yml
       env:
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
-        SLACK_NOTIFICATIONS_CHANNEL: "#lucene"
+        SLACK_NOTIFICATIONS_CHANNEL: "#lucene-ci"
         SLACK_NOTIFICATIONS_ALL_BRANCHES: "true"
       default_branch: lucene_snapshot
       teams:
@@ -210,7 +210,7 @@ spec:
       pipeline_file: .buildkite/pipelines/lucene-snapshot/run-tests.yml
       env:
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
-        SLACK_NOTIFICATIONS_CHANNEL: "#lucene"
+        SLACK_NOTIFICATIONS_CHANNEL: "#lucene-ci"
         SLACK_NOTIFICATIONS_ALL_BRANCHES: "true"
       branch_configuration: lucene_snapshot
       default_branch: lucene_snapshot

docs/build.gradle

@@ -855,6 +855,9 @@ buildRestTests.setups['library'] = '''
 
 '''
 buildRestTests.setups['sensor_rollup_job'] = '''
+  - requires:
+      test_runner_features: [ "allowed_warnings" ]
+
   - do:
       indices.create:
         index: dummy-rollup-index
@@ -885,9 +888,10 @@ buildRestTests.setups['sensor_rollup_job'] = '''
               node:
                 type: keyword
   - do:
-      raw:
-        method: PUT
-        path: _rollup/job/sensor
+      allowed_warnings:
+        - "The rollup functionality will be removed in Elasticsearch 10.0. See docs for more information."
+      rollup.put_job:
+        id: sensor
         body:  >
             {
                 "index_pattern": "sensor-*",
@@ -917,6 +921,9 @@ buildRestTests.setups['sensor_rollup_job'] = '''
             }
 '''
 buildRestTests.setups['sensor_started_rollup_job'] = '''
+  - requires:
+      test_runner_features: [ "allowed_warnings" ]
+
   - do:
       indices.create:
         index: dummy-rollup-index
@@ -966,9 +973,10 @@ buildRestTests.setups['sensor_started_rollup_job'] = '''
           {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"}
 
   - do:
-      raw:
-        method: PUT
-        path: _rollup/job/sensor
+      allowed_warnings:
+        - "The rollup functionality will be removed in Elasticsearch 10.0. See docs for more information."
+      rollup.put_job:
+        id: sensor
         body:  >
             {
                 "index_pattern": "sensor-*",
@@ -997,9 +1005,10 @@ buildRestTests.setups['sensor_started_rollup_job'] = '''
                 ]
             }
   - do:
-      raw:
-        method: POST
-        path: _rollup/job/sensor/_start
+      allowed_warnings:
+        - "The rollup functionality will be removed in Elasticsearch 10.0. See docs for more information."
+      rollup.start_job:
+        id: sensor
 '''
 
 buildRestTests.setups['sensor_index'] = '''

docs/changelog/113131.yaml

@@ -0,0 +1,10 @@
+pr: 113131
+summary: Emit deprecation warning when executing one of the rollup APIs
+area: Rollup
+type: deprecation
+issues: []
+deprecation:
+  title: Emit deprecation warning when executing one of the rollup APIs
+  area: Rollup
+  details: Rollup is already deprecated since 8.11.0 via documentation and since 8.15.0 it is no longer possible to create new rollup jobs in clusters without rollup usage. This change updates the rollup APIs to emit a deprecation warning.
+  impact: Returning a deprecation warning when using one of the rollup APIs.

docs/changelog/120168.yaml

@@ -0,0 +1,5 @@
+pr: 121074
+summary: Implement a `MetricsAware` interface
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/121074.yaml

@@ -0,0 +1,5 @@
+pr: 121119
+summary: Revert "Reduce Data Loss in System Indices Migration"
+area: Infra/Core
+type: bug
+issues: []

docs/changelog/121119.yaml

@@ -6,9 +6,14 @@
 
 preview::[]
 
-The time series aggregation queries data created using a time series index. This is typically data such as metrics
+The time series aggregation queries data created using a <<tsds,Time series data stream (TSDS)>>. This is typically data such as metrics
 or other data streams with a time component, and requires creating an index using the time series mode.
 
+[NOTE]
+====
+Refer to the <<differences-from-regular-data-stream, TSDS documentation>> to learn more about the key differences from regular data streams.
+====
+
 //////////////////////////
 
 Creating a time series mapping

docs/reference/aggregations/bucket/time-series-aggregation.asciidoc

@@ -254,6 +254,16 @@ as they contain data essential to the operation of the system.
 IMPORTANT: Direct access to system indices is deprecated and
 will no longer be allowed in a future major version.
 
+To view system indices within cluster:
+
+[source,console]
+--------------------------------------------------
+GET _cluster/state/metadata?filter_path=metadata.indices.*.system
+--------------------------------------------------
+
+WARNING: When overwriting current cluster state, system indices should be restored
+as part of their {ref}/snapshot-restore.html#feature-state[feature state].
+
 [discrete]
 [[api-conventions-parameters]]
 === Parameters

docs/reference/api-conventions.asciidoc

@@ -307,23 +307,19 @@ POST test/_update/1
 [discrete]
 ===== Upsert
 
-If the document does not already exist, the contents of the `upsert` element
-are inserted as a new document. If the document exists, the
-`script` is executed:
+An upsert operation lets you update an existing document or insert a new one if it doesn't exist, in a single request.
+
+In this example, if the product with ID `1` exists, its price will be updated to `100`. If the product does not exist, a new document with ID `1` and a price of `50` will be inserted.
 
 [source,console]
 ----
-POST test/_update/1
+POST /test/_update/1
 {
-  "script": {
-    "source": "ctx._source.counter += params.count",
-    "lang": "painless",
-    "params": {
-      "count": 4
-    }
+  "doc": {
+    "product_price": 100
   },
   "upsert": {
-    "counter": 1
+    "product_price": 50
   }
 }
 ----

docs/reference/docs/update.asciidoc

@@ -0,0 +1,142 @@
+[[indices-create-index-from-source]]
+=== Create index from source API
+++++
+<titleabbrev>Create index from source</titleabbrev>
+++++
+
+.New API reference
+[sidebar]
+--
+For the most up-to-date API details, refer to {api-es}/group/endpoint-indices[Index APIs].
+--
+
+[[indices-create-index-from-source-api-request]]
+==== {api-request-title}
+
+`PUT /_create_from/<source>/<dest>`
+
+`POST/_create_from/<source>/<dest>`
+
+[[indices-create-index-from-source-api-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have the `manage`
+<<privileges-list-indices,index privilege>> for the index.
+
+[[indices-create-index-from-source-api-desc]]
+==== {api-description-title}
+This api allows you to add a new index to an {es} cluster, using an existing source index as a basis for the new index.
+The settings and mappings from the source index will copied over to the destination index.  You can also provide
+override settings and mappings which will be combined with the source settings and mappings when creating the
+destination index.
+
+[[indices-create-index-from-source-api-path-params]]
+==== {api-path-parms-title}
+
+`<source>`::
+(Required, string) Name of the existing source index which will be used as a basis.
+
+`<dest>`::
+(Required, string) Name of the destination index which will be created.
+
+
+[role="child_attributes"]
+[[indices-create-index-from-source-api-request-body]]
+==== {api-request-body-title}
+
+`settings_override`::
+(Optional, <<index-modules-settings,index setting object>>) Settings which override the source settings.
+
+`mappings_override`::
+(Optional, <<mapping,mapping object>>) Mappings which override the source mappings.
+
+`remove_index_blocks`::
+(Optional, boolean) Filter out any index blocks from the source index when creating the destination index.
+Defaults to `true`.
+
+[[indices-create-index-from-source-api-example]]
+==== {api-examples-title}
+
+Start by creating a source index that we'll copy using this API.
+
+[source,console]
+--------------------------------------------------
+PUT /my-index
+{
+  "settings": {
+    "index": {
+      "number_of_shards": 3,
+      "blocks.write": true
+    }
+  },
+  "mappings": {
+    "properties": {
+        "field1": { "type": "text" }
+    }
+  }
+}
+--------------------------------------------------
+// TESTSETUP
+
+Now we create a destination index from the source index. This new index will have the same mappings and settings
+as the source index.
+
+[source,console]
+--------------------------------------------------
+POST _create_from/my-index/my-new-index
+--------------------------------------------------
+
+
+Alternatively, we could override some of the source's settings and mappings. This will use the source settings
+and mappings as a basis and combine these with the overrides to create the destination settings and mappings.
+
+[source,console]
+--------------------------------------------------
+POST _create_from/my-index/my-new-index
+{
+  "settings_override": {
+    "index": {
+      "number_of_shards": 5
+    }
+  },
+  "mappings_override": {
+    "properties": {
+        "field2": { "type": "boolean" }
+    }
+  }
+}
+--------------------------------------------------
+
+Since the destination index is empty, we very likely will want to write into the index after creation.
+This would not be possible if the source index contains an <<index-block-settings,index write block>> which is copied over to the destination index.
+One way to handle this is to remove the index write block using a settings override. For example, the following
+settings override removes all index blocks.
+
+
+[source,console]
+--------------------------------------------------
+POST _create_from/my-index/my-new-index
+{
+  "settings_override": {
+    "index": {
+      "blocks.write": null,
+      "blocks.read": null,
+      "blocks.read_only": null,
+      "blocks.read_only_allow_delete": null,
+      "blocks.metadata": null
+    }
+  }
+}
+--------------------------------------------------
+
+Since this is a common scenario, index blocks are actually removed by default. This is controlled with the parameter
+`remove_index_blocks`, which defaults to `true`. If we want the destination index to contains the index blocks from
+the source index, we can do the following:
+
+[source,console]
+--------------------------------------------------
+POST _create_from/my-index/my-new-index
+{
+  "remove_index_blocks": false
+}
+--------------------------------------------------