failure store backport (#132769)

Author: shainaraskas

docs/build.gradle

@@ -2013,6 +2013,101 @@ setups['setup-snapshots'] = setups['setup-repository'] + '''
           refresh: true
           body: |'''
 
+  buildRestTests.setups['failure-store-recipes-1'] = '''
+  - do:
+      ingest.put_pipeline:
+        id: ingest-step-1
+        body: >
+          {
+            "processors": [
+              {
+                "remove": {
+                  "field": "important.info"
+                }
+              },
+              {
+                "pipeline": {
+                  "name": "ingest-step-2"
+                }
+              }
+            ]
+          }
+
+  - do:
+      ingest.put_pipeline:
+        id: ingest-step-2
+        body: >
+          {
+            "processors": [
+              {
+                "set": {
+                  "field": "copy.info",
+                  "copy_from": "important.info"
+                }
+              }
+            ]
+          }
+
+  - do:
+      indices.put_index_template:
+        name: my-datastream-ingest-template
+        body:
+          index_patterns: my-datastream-ingest
+          data_stream: {}
+          template:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 0
+              index:
+                default_pipeline: "ingest-step-1"
+            data_stream_options:
+              failure_store:
+                enabled: true'''
+
+  buildRestTests.teardowns['failure-store-recipes-1-td'] = '''
+  - do:
+      indices.delete_data_stream:
+        name: my-datastream-ingest
+
+  - do:
+      indices.delete_index_template:
+        name: my-datastream-ingest-template
+
+  - do:
+      ingest.delete_pipeline:
+        id: ingest-step-1
+
+  - do:
+      ingest.delete_pipeline:
+        id: ingest-step-2
+'''
+
+buildRestTests.setups['failure-store-recipes-2'] = '''
+  - do:
+      indices.put_index_template:
+        name: my-datastream-ingest-template
+        body:
+          index_patterns: my-datastream-ingest
+          data_stream: {}
+          template:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 0
+              index:
+                default_pipeline: "complicated-processor"
+            data_stream_options:
+              failure_store:
+                enabled: true'''
+
+buildRestTests.teardowns['failure-store-recipes-2-td'] = '''
+  - do:
+      indices.delete_data_stream:
+        name: my-datastream-ingest
+
+  - do:
+      indices.delete_index_template:
+        name: my-datastream-ingest-template
+'''
 
   for (int i = 100; i < 200; i++) {
     def value = i

docs/reference/api-conventions.asciidoc

@@ -264,6 +264,48 @@ GET _cluster/state/metadata?filter_path=metadata.indices.*.system
 WARNING: When overwriting current cluster state, system indices should be restored
 as part of their {ref}/snapshot-restore.html#feature-state[feature state].
 
+[discrete]
+[[node-spec-convention]]
+==== Node specification
+
+Some cluster-level APIs may operate on a subset of the nodes which can be specified with node filters. For example, task management, node stats, and node info APIs can all report results from a filtered set of nodes rather than from all nodes. <<cluster-nodes,Learn more>>.
+
+[discrete]
+[[component-selectors]]
+==== Component selectors
+
+A data stream component is a logical grouping of indices that help organize data inside a data stream. All data streams contain a `data` component by default. The `data` component comprises the data stream's backing indices. When searching, managing, or indexing into a data stream, the data component is what you are interacting with by default.
+
+Some data stream features are exposed as additional components alongside its `data` component. These other components are comprised of separate sets of backing indices. These additional components store supplemental data independent of the data stream's regular backing indices. An example of another component is the `failures` component exposed by the data stream <<failure-store, failure store>> feature, which captures documents that fail to be ingested in a separate set of backing indices on the data stream.
+
+Some APIs that accept a `<data-stream>`, `<index>`, or `<target>` request path parameter also support selector syntax which defines which component on a data stream the API should operate on. To use a selector, it is appended to the index or data stream name. Selectors can be combined with other index pattern syntax like <<api-date-math-index-names,date math>> and wildcards.
+
+There are two selector suffixes supported by Elasticsearch APIs:
+
+`::data`::
+  Refers to a data stream's backing indices containing regular data. Data streams always contain a data component.
+`::failures`::
+  This component refers to the internal indices used for a data stream's <<failure-store, failure store>>.
+
+As an example, <<search>>, <<search-field-caps,field capabilities>>, and <<indices-stats,index stats>> APIs can all report results from a different component rather than from the default data.
+
+[source,console]
+----
+# Search a data stream normally
+GET my-data-stream/_search
+
+# Search a data stream's failure data if present
+GET my-data-stream::failures/_search
+
+# Syntax can be combined with other index pattern syntax (wildcards, multi-target, date math, cross cluster search, etc)
+GET logs-*::failures/_search
+GET logs-*::data,logs-*::failures/_count
+GET remote-cluster:logs-*-*::failures/_search
+GET *::data,*::failures,-logs-rdbms-*::failures/_stats
+GET <logs-{now/d}>::failures/_search
+----
+// TEST[skip:backport]
+
 [discrete]
 [[api-conventions-parameters]]
 === Parameters

docs/reference/data-streams/data-streams.asciidoc

@@ -159,3 +159,4 @@ include::change-mappings-and-settings.asciidoc[]
 include::tsds.asciidoc[]
 include::logs.asciidoc[]
 include::lifecycle/index.asciidoc[]
+include::failure-store.asciidoc[]