diff --git a/docs/changelog/114128.yaml b/docs/changelog/114128.yaml new file mode 100644 index 0000000000000..721649d0d6fe0 --- /dev/null +++ b/docs/changelog/114128.yaml @@ -0,0 +1,5 @@ +pr: 114128 +summary: Adding `index_template_substitutions` to the simulate ingest API +area: Ingest Node +type: enhancement +issues: [] diff --git a/docs/reference/indices/put-index-template.asciidoc b/docs/reference/indices/put-index-template.asciidoc index 772bd51afdce8..36fc66ecb90b8 100644 --- a/docs/reference/indices/put-index-template.asciidoc +++ b/docs/reference/indices/put-index-template.asciidoc @@ -85,6 +85,8 @@ include::{docdir}/rest-api/common-parms.asciidoc[tag=master-timeout] [[put-index-template-api-request-body]] ==== {api-request-body-title} +// tag::request-body[] + `composed_of`:: (Optional, array of strings) An ordered list of component template names. Component templates are merged in the order @@ -102,7 +104,7 @@ See <>. + .Properties of `data_stream` [%collapsible%open] -==== +===== `allow_custom_routing`:: (Optional, Boolean) If `true`, the data stream supports <>. Defaults to `false`. @@ -117,7 +119,7 @@ See <>. + If `time_series`, each backing index has an `index.mode` index setting of `time_series`. -==== +===== `index_patterns`:: (Required, array of strings) @@ -146,7 +148,7 @@ Template to be applied. It may optionally include an `aliases`, `mappings`, or + .Properties of `template` [%collapsible%open] -==== +===== `aliases`:: (Optional, object of objects) Aliases to add. + @@ -161,7 +163,7 @@ include::{es-ref-dir}/indices/create-index.asciidoc[tag=aliases-props] include::{docdir}/rest-api/common-parms.asciidoc[tag=mappings] include::{docdir}/rest-api/common-parms.asciidoc[tag=settings] -==== +===== `version`:: (Optional, integer) @@ -174,6 +176,7 @@ Marks this index template as deprecated. When creating or updating a non-deprecated index template that uses deprecated components, {es} will emit a deprecation warning. // end::index-template-api-body[] +// end::request-body[] [[put-index-template-api-example]] ==== {api-examples-title} diff --git a/docs/reference/ingest/apis/simulate-ingest.asciidoc b/docs/reference/ingest/apis/simulate-ingest.asciidoc index ac6da515402bb..1bee03ea3e58a 100644 --- a/docs/reference/ingest/apis/simulate-ingest.asciidoc +++ b/docs/reference/ingest/apis/simulate-ingest.asciidoc @@ -102,6 +102,12 @@ POST /_ingest/_simulate } } } + }, + "index_template_substitutions": { <3> + "my-index-template": { + "index_patterns": ["my-index-*"], + "composed_of": ["component_template_1", "component_template_2"] + } } } ---- @@ -109,6 +115,8 @@ POST /_ingest/_simulate <1> This replaces the existing `my-pipeline` pipeline with the contents given here for the duration of this request. <2> This replaces the existing `my-component-template` component template with the contents given here for the duration of this request. These templates can be used to change the pipeline(s) used, or to modify the mapping that will be used to validate the result. +<3> This replaces the existing `my-index-template` index template with the contents given here for the duration of this request. +These templates can be used to change the pipeline(s) used, or to modify the mapping that will be used to validate the result. [[simulate-ingest-api-request]] ==== {api-request-title} @@ -225,6 +233,19 @@ include::{es-ref-dir}/indices/put-component-template.asciidoc[tag=template] ==== +`index_template_substitutions`:: +(Optional, map of strings to objects) +Map of index template names to substitute index template definition objects. ++ +.Properties of index template definition objects +[%collapsible%open] + +==== + +include::{es-ref-dir}/indices/put-index-template.asciidoc[tag=request-body] + +==== + [[simulate-ingest-api-example]] ==== {api-examples-title} diff --git a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml index b4672b1d8924d..18eb401aaa0fe 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml +++ b/qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml @@ -371,7 +371,7 @@ setup: template: settings: index: - default_pipeline: "foo_pipeline" + default_pipeline: "foo-pipeline" - do: allowed_warnings: @@ -523,7 +523,7 @@ setup: template: settings: index: - default_pipeline: "foo_pipeline" + default_pipeline: "foo-pipeline" - do: allowed_warnings: @@ -807,3 +807,412 @@ setup: - match: { docs.0.doc._source.foo: "FOO" } - match: { docs.0.doc.executed_pipelines: ["foo-pipeline-2"] } - not_exists: docs.0.doc.error + +--- +"Test ingest simulate with index template substitutions": + + - skip: + features: + - headers + - allowed_warnings + + - requires: + cluster_features: ["simulate.index.template.substitutions"] + reason: "ingest simulate index template substitutions added in 8.16" + + - do: + headers: + Content-Type: application/json + ingest.put_pipeline: + id: "foo-pipeline" + body: > + { + "processors": [ + { + "set": { + "field": "foo", + "value": true + } + } + ] + } + - match: { acknowledged: true } + + - do: + cluster.put_component_template: + name: settings_template + body: + template: + settings: + index: + default_pipeline: "foo-pipeline" + + - do: + cluster.put_component_template: + name: mappings_template + body: + template: + mappings: + dynamic: strict + properties: + foo: + type: keyword + + - do: + allowed_warnings: + - "index template [test-composable-1] has index patterns [foo*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-1] will take precedence during new index creation" + indices.put_index_template: + name: test-composable-1 + body: + index_patterns: + - foo* + composed_of: + - mappings_template + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": "FOO" + } + } + ], + "component_template_substitutions": { + "settings_template": { + "template": { + "settings": { + "index": { + "default_pipeline": null + } + } + } + } + }, + "index_template_substitutions": { + "foo_index_template": { + "index_patterns":[ + "foo*" + ], + "composed_of": ["settings_template"] + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: [] } + - not_exists: docs.0.doc.error + + - do: + indices.create: + index: foo-1 + - match: { acknowledged: true } + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": "FOO" + } + } + ], + "component_template_substitutions": { + "settings_template": { + "template": { + "settings": { + "index": { + "default_pipeline": null + } + } + } + } + }, + "index_template_substitutions": { + "foo_index_template": { + "index_patterns":[ + "foo*" + ], + "composed_of": ["settings_template", "mappings_template"] + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: [] } + - not_exists: docs.0.doc.error + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: foo-1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "foo": "FOO" + } + } + ], + "component_template_substitutions": { + "mappings_template": { + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "boolean" + } + } + } + } + } + }, + "index_template_substitutions": { + "foo_index_template": { + "index_patterns":[ + "foo*" + ], + "composed_of": ["settings_template", "mappings_template"] + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "foo-1" } + - match: { docs.0.doc._source.foo: true } + - match: { docs.0.doc.executed_pipelines: ["foo-pipeline"] } + - not_exists: docs.0.doc.error + +--- +"Test ingest simulate with index template substitutions for data streams": + # In this test, we make sure that when the index template is a data stream template, simulate ingest works the same whether the data + # stream has been created or not -- either way, we expect it to use the template rather than the data stream / index mappings and settings. + + - skip: + features: + - headers + - allowed_warnings + + - requires: + cluster_features: ["simulate.index.template.substitutions"] + reason: "ingest simulate component template substitutions added in 8.16" + + - do: + headers: + Content-Type: application/json + ingest.put_pipeline: + id: "foo-pipeline" + body: > + { + "processors": [ + { + "set": { + "field": "foo", + "value": true + } + } + ] + } + - match: { acknowledged: true } + + - do: + cluster.put_component_template: + name: mappings_template + body: + template: + mappings: + dynamic: strict + properties: + foo: + type: boolean + + - do: + cluster.put_component_template: + name: settings_template + body: + template: + settings: + index: + default_pipeline: "foo-pipeline" + + - do: + allowed_warnings: + - "index template [test-composable-1] has index patterns [foo*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-1] will take precedence during new index creation" + indices.put_index_template: + name: test-composable-1 + body: + index_patterns: + - foo* + composed_of: + - mappings_template + - settings_template + + - do: + allowed_warnings: + - "index template [my-template-1] has index patterns [simple-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template-1] will take precedence during new index creation" + indices.put_index_template: + name: my-template-1 + body: + index_patterns: [simple-data-stream1] + composed_of: + - mappings_template + - settings_template + data_stream: {} + + # Here we replace my-template-1 with a substitute version that uses the settings_template_2 and mappings_template_2 templates defined in + # this request, and foo-pipeline-2 defined in this request. + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: simple-data-stream1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "@timestamp": 1234, + "foo": false + } + } + ], + "pipeline_substitutions": { + "foo-pipeline-2": { + "processors": [ + { + "set": { + "field": "foo", + "value": "FOO" + } + } + ] + } + }, + "component_template_substitutions": { + "settings_template_2": { + "template": { + "settings": { + "index": { + "default_pipeline": "foo-pipeline-2" + } + } + } + }, + "mappings_template_2": { + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "keyword" + } + } + } + } + } + }, + "index_template_substitutions": { + "my-template-1": { + "index_patterns": ["simple-data-stream1"], + "composed_of": ["settings_template_2", "mappings_template_2"], + "data_stream": {} + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "simple-data-stream1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: ["foo-pipeline-2"] } + - not_exists: docs.0.doc.error + + - do: + indices.create_data_stream: + name: simple-data-stream1 + - is_true: acknowledged + + - do: + cluster.health: + wait_for_status: yellow + + - do: + headers: + Content-Type: application/json + simulate.ingest: + index: simple-data-stream1 + body: > + { + "docs": [ + { + "_id": "asdf", + "_source": { + "@timestamp": 1234, + "foo": false + } + } + ], + "pipeline_substitutions": { + "foo-pipeline-2": { + "processors": [ + { + "set": { + "field": "foo", + "value": "FOO" + } + } + ] + } + }, + "component_template_substitutions": { + "settings_template_2": { + "template": { + "settings": { + "index": { + "default_pipeline": "foo-pipeline-2" + } + } + } + }, + "mappings_template_2": { + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "foo": { + "type": "keyword" + } + } + } + } + } + }, + "index_template_substitutions": { + "my-template-1": { + "index_patterns": ["simple-data-stream1"], + "composed_of": ["settings_template_2", "mappings_template_2"], + "data_stream": {} + } + } + } + - length: { docs: 1 } + - match: { docs.0.doc._index: "simple-data-stream1" } + - match: { docs.0.doc._source.foo: "FOO" } + - match: { docs.0.doc.executed_pipelines: ["foo-pipeline-2"] } + - not_exists: docs.0.doc.error diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java index 91674b7ce9050..af99a0344e030 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionIT.java @@ -59,7 +59,7 @@ public void testMappingValidationIndexExists() { } """; indicesAdmin().create(new CreateIndexRequest(indexName).mapping(mapping)).actionGet(); - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -90,7 +90,7 @@ public void testMappingValidationIndexExists() { } @SuppressWarnings("unchecked") - public void testMappingValidationIndexExistsWithComponentTemplate() throws IOException { + public void testMappingValidationIndexExistsTemplateSubstitutions() throws IOException { /* * This test simulates a BulkRequest of two documents into an existing index. Then we make sure the index contains no documents, and * that the index's mapping in the cluster state has not been updated with the two new field. With the mapping from the template @@ -122,16 +122,19 @@ public void testMappingValidationIndexExistsWithComponentTemplate() throws IOExc .indexPatterns(List.of("my-index-*")) .componentTemplates(List.of("test-component-template")) .build(); - TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request("test"); + final String indexTemplateName = "test-index-template"; + TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request( + indexTemplateName + ); request.indexTemplate(composableIndexTemplate); client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); String indexName = "my-index-1"; // First, run before the index is created: - assertMappingsUpdatedFromComponentTemplateSubstitutions(indexName); + assertMappingsUpdatedFromComponentTemplateSubstitutions(indexName, indexTemplateName); // Now, create the index and make sure the component template substitutions work the same: indicesAdmin().create(new CreateIndexRequest(indexName)).actionGet(); - assertMappingsUpdatedFromComponentTemplateSubstitutions(indexName); + assertMappingsUpdatedFromComponentTemplateSubstitutions(indexName, indexTemplateName); // Now make sure nothing was actually changed: indicesAdmin().refresh(new RefreshRequest(indexName)).actionGet(); SearchResponse searchResponse = client().search(new SearchRequest(indexName)).actionGet(); @@ -143,7 +146,7 @@ public void testMappingValidationIndexExistsWithComponentTemplate() throws IOExc assertThat(fields.size(), equalTo(1)); } - private void assertMappingsUpdatedFromComponentTemplateSubstitutions(String indexName) { + private void assertMappingsUpdatedFromComponentTemplateSubstitutions(String indexName, String indexTemplateName) { IndexRequest indexRequest1 = new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -156,7 +159,7 @@ private void assertMappingsUpdatedFromComponentTemplateSubstitutions(String inde """, XContentType.JSON).id(randomUUID()); { // First we use the original component template, and expect a failure in the second document: - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of()); bulkRequest.add(indexRequest1); bulkRequest.add(indexRequest2); BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); @@ -188,7 +191,42 @@ private void assertMappingsUpdatedFromComponentTemplateSubstitutions(String inde ) ) ) - ) + ), + Map.of() + ); + bulkRequest.add(indexRequest1); + bulkRequest.add(indexRequest2); + BulkResponse response = client().execute(new ActionType(SimulateBulkAction.NAME), bulkRequest).actionGet(); + assertThat(response.getItems().length, equalTo(2)); + assertThat(response.getItems()[0].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[0].getResponse()).getException()); + assertThat(response.getItems()[1].getResponse().getResult(), equalTo(DocWriteResponse.Result.CREATED)); + assertNull(((SimulateIndexResponse) response.getItems()[1].getResponse()).getException()); + } + + { + /* + * Now we substitute a "test-component-template-2" that defines both fields, and an index template that pulls it in, so we + * expect no exception: + */ + BulkRequest bulkRequest = new SimulateBulkRequest( + Map.of(), + Map.of( + "test-component-template-2", + Map.of( + "template", + Map.of( + "mappings", + Map.of( + "dynamic", + "strict", + "properties", + Map.of("foo1", Map.of("type", "text"), "foo3", Map.of("type", "text")) + ) + ) + ) + ), + Map.of(indexTemplateName, Map.of("index_patterns", List.of(indexName), "composed_of", List.of("test-component-template-2"))) ); bulkRequest.add(indexRequest1); bulkRequest.add(indexRequest2); @@ -207,7 +245,7 @@ public void testMappingValidationIndexDoesNotExistsNoTemplate() { * mapping-less "random-index-template" created by the parent class), so we expect no mapping validation failure. */ String indexName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -254,7 +292,7 @@ public void testMappingValidationIndexDoesNotExistsV2Template() throws IOExcepti request.indexTemplate(composableIndexTemplate); client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -286,7 +324,7 @@ public void testMappingValidationIndexDoesNotExistsV1Template() { indicesAdmin().putTemplate( new PutIndexTemplateRequest("test-template").patterns(List.of("my-index-*")).mapping("foo1", "type=integer") ).actionGet(); - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -340,7 +378,7 @@ public void testMappingValidationIndexDoesNotExistsDataStream() throws IOExcepti client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); { // First, try with no @timestamp to make sure we're picking up data-stream-specific templates - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "foo1": "baz" @@ -366,7 +404,7 @@ public void testMappingValidationIndexDoesNotExistsDataStream() throws IOExcepti } { // Now with @timestamp - BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + BulkRequest bulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of()); bulkRequest.add(new IndexRequest(indexName).source(""" { "@timestamp": "2024-08-27", diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 2095ba47ee377..78fddad603cab 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -237,6 +237,7 @@ static TransportVersion def(int id) { public static final TransportVersion DATE_TIME_DOC_VALUES_LOCALES = def(8_761_00_0); public static final TransportVersion FAST_REFRESH_RCO = def(8_762_00_0); public static final TransportVersion TEXT_SIMILARITY_RERANKER_QUERY_REWRITE = def(8_763_00_0); + public static final TransportVersion SIMULATE_INDEX_TEMPLATES_SUBSTITUTIONS = def(8_764_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java index af1782ac1ade3..78e603fba9be0 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkFeatures.java @@ -15,11 +15,17 @@ import java.util.Set; import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS; +import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_INDEX_TEMPLATE_SUBSTITUTIONS; import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_VALIDATION; import static org.elasticsearch.action.bulk.TransportSimulateBulkAction.SIMULATE_MAPPING_VALIDATION_TEMPLATES; public class BulkFeatures implements FeatureSpecification { public Set getFeatures() { - return Set.of(SIMULATE_MAPPING_VALIDATION, SIMULATE_MAPPING_VALIDATION_TEMPLATES, SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS); + return Set.of( + SIMULATE_MAPPING_VALIDATION, + SIMULATE_MAPPING_VALIDATION_TEMPLATES, + SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS, + SIMULATE_INDEX_TEMPLATE_SUBSTITUTIONS + ); } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java index 558901f102299..f62b2f48fa2fd 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequest.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.support.replication.ReplicationRequest; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -506,6 +507,10 @@ public Map getComponentTemplateSubstitutions() throws return Map.of(); } + public Map getIndexTemplateSubstitutions() throws IOException { + return Map.of(); + } + record IncrementalState(Map shardLevelFailures, boolean indexingPressureAccounted) implements Writeable { static final IncrementalState EMPTY = new IncrementalState(Collections.emptyMap(), false); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java index 3cc7fa12733bf..6fa22151396df 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/SimulateBulkRequest.java @@ -11,6 +11,7 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.XContentHelper; @@ -22,8 +23,8 @@ import java.util.Map; /** - * This extends BulkRequest with support for providing substitute pipeline definitions and component template definitions. In a user - * request, the substitutions will look something like this: + * This extends BulkRequest with support for providing substitute pipeline definitions, component template definitions, and index template + * substitutions. In a user request, the substitutions will look something like this: * * "pipeline_substitutions": { * "my-pipeline-1": { @@ -72,6 +73,16 @@ * } * } * } + * }, + * "index_template_substitutions": { + * "my-index-template-1": { + * "template": { + * "index_patterns": ["foo*", "bar*"] + * "composed_of": [ + * "component-template-1", + * "component-template-2" + * ] + * } * } * } * @@ -82,6 +93,7 @@ public class SimulateBulkRequest extends BulkRequest { private final Map> pipelineSubstitutions; private final Map> componentTemplateSubstitutions; + private final Map> indexTemplateSubstitutions; /** * @param pipelineSubstitutions The pipeline definitions that are to be used in place of any pre-existing pipeline definitions with @@ -89,14 +101,18 @@ public class SimulateBulkRequest extends BulkRequest { * parsed by XContentHelper.convertToMap(). * @param componentTemplateSubstitutions The component template definitions that are to be used in place of any pre-existing * component template definitions with the same name. + * @param indexTemplateSubstitutions The index template definitions that are to be used in place of any pre-existing + * index template definitions with the same name. */ public SimulateBulkRequest( @Nullable Map> pipelineSubstitutions, - @Nullable Map> componentTemplateSubstitutions + @Nullable Map> componentTemplateSubstitutions, + @Nullable Map> indexTemplateSubstitutions ) { super(); this.pipelineSubstitutions = pipelineSubstitutions; this.componentTemplateSubstitutions = componentTemplateSubstitutions; + this.indexTemplateSubstitutions = indexTemplateSubstitutions; } @SuppressWarnings("unchecked") @@ -108,6 +124,11 @@ public SimulateBulkRequest(StreamInput in) throws IOException { } else { componentTemplateSubstitutions = Map.of(); } + if (in.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_INDEX_TEMPLATES_SUBSTITUTIONS)) { + this.indexTemplateSubstitutions = (Map>) in.readGenericValue(); + } else { + indexTemplateSubstitutions = Map.of(); + } } @Override @@ -117,6 +138,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_COMPONENT_TEMPLATES_SUBSTITUTIONS)) { out.writeGenericValue(componentTemplateSubstitutions); } + if (out.getTransportVersion().onOrAfter(TransportVersions.SIMULATE_INDEX_TEMPLATES_SUBSTITUTIONS)) { + out.writeGenericValue(indexTemplateSubstitutions); + } } public Map> getPipelineSubstitutions() { @@ -140,6 +164,18 @@ public Map getComponentTemplateSubstitutions() throws return result; } + @Override + public Map getIndexTemplateSubstitutions() throws IOException { + if (indexTemplateSubstitutions == null) { + return Map.of(); + } + Map result = new HashMap<>(indexTemplateSubstitutions.size()); + for (Map.Entry> rawEntry : indexTemplateSubstitutions.entrySet()) { + result.put(rawEntry.getKey(), convertRawTemplateToIndexTemplate(rawEntry.getValue())); + } + return result; + } + private static ComponentTemplate convertRawTemplateToComponentTemplate(Map rawTemplate) throws IOException { ComponentTemplate componentTemplate; try (var parser = XContentHelper.mapToXContentParser(XContentParserConfiguration.EMPTY, rawTemplate)) { @@ -148,9 +184,21 @@ private static ComponentTemplate convertRawTemplateToComponentTemplate(Map rawTemplate) throws IOException { + ComposableIndexTemplate indexTemplate; + try (var parser = XContentHelper.mapToXContentParser(XContentParserConfiguration.EMPTY, rawTemplate)) { + indexTemplate = ComposableIndexTemplate.parse(parser); + } + return indexTemplate; + } + @Override public BulkRequest shallowClone() { - BulkRequest bulkRequest = new SimulateBulkRequest(pipelineSubstitutions, componentTemplateSubstitutions); + BulkRequest bulkRequest = new SimulateBulkRequest( + pipelineSubstitutions, + componentTemplateSubstitutions, + indexTemplateSubstitutions + ); bulkRequest.setRefreshPolicy(getRefreshPolicy()); bulkRequest.waitForActiveShards(waitForActiveShards()); bulkRequest.timeout(timeout()); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java index 8c6565e52daa7..111e4d72c57c6 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportAbstractBulkAction.java @@ -25,6 +25,7 @@ import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.Writeable; @@ -183,7 +184,9 @@ private boolean applyPipelines(Task task, BulkRequest bulkRequest, Executor exec boolean hasIndexRequestsWithPipelines = false; final Metadata metadata; Map componentTemplateSubstitutions = bulkRequest.getComponentTemplateSubstitutions(); - if (bulkRequest.isSimulated() && componentTemplateSubstitutions.isEmpty() == false) { + Map indexTemplateSubstitutions = bulkRequest.getIndexTemplateSubstitutions(); + if (bulkRequest.isSimulated() + && (componentTemplateSubstitutions.isEmpty() == false || indexTemplateSubstitutions.isEmpty() == false)) { /* * If this is a simulated request, and there are template substitutions, then we want to create and use a new metadata that has * those templates. That is, we want to add the new templates (which will replace any that already existed with the same name), @@ -197,6 +200,12 @@ private boolean applyPipelines(Task task, BulkRequest bulkRequest, Executor exec updatedComponentTemplates.putAll(componentTemplateSubstitutions); simulatedMetadataBuilder.componentTemplates(updatedComponentTemplates); } + if (indexTemplateSubstitutions.isEmpty() == false) { + Map updatedIndexTemplates = new HashMap<>(); + updatedIndexTemplates.putAll(clusterService.state().metadata().templatesV2()); + updatedIndexTemplates.putAll(indexTemplateSubstitutions); + simulatedMetadataBuilder.indexTemplates(updatedIndexTemplates); + } /* * We now remove the index from the simulated metadata to force the templates to be used. Note that simulated requests are * always index requests -- no other type of request is supported. diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index 713116c4cf98e..d7c555879c00f 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -17,6 +17,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; @@ -73,6 +74,7 @@ public class TransportSimulateBulkAction extends TransportAbstractBulkAction { public static final NodeFeature SIMULATE_COMPONENT_TEMPLATE_SUBSTITUTIONS = new NodeFeature( "simulate.component.template.substitutions" ); + public static final NodeFeature SIMULATE_INDEX_TEMPLATE_SUBSTITUTIONS = new NodeFeature("simulate.index.template.substitutions"); private final IndicesService indicesService; private final NamedXContentRegistry xContentRegistry; private final Set indexSettingProviders; @@ -119,11 +121,12 @@ protected void doInternalExecute( : "TransportSimulateBulkAction should only ever be called with a SimulateBulkRequest but got a " + bulkRequest.getClass(); final AtomicArray responses = new AtomicArray<>(bulkRequest.requests.size()); Map componentTemplateSubstitutions = bulkRequest.getComponentTemplateSubstitutions(); + Map indexTemplateSubstitutions = bulkRequest.getIndexTemplateSubstitutions(); for (int i = 0; i < bulkRequest.requests.size(); i++) { DocWriteRequest docRequest = bulkRequest.requests.get(i); assert docRequest instanceof IndexRequest : "TransportSimulateBulkAction should only ever be called with IndexRequests"; IndexRequest request = (IndexRequest) docRequest; - Exception mappingValidationException = validateMappings(componentTemplateSubstitutions, request); + Exception mappingValidationException = validateMappings(componentTemplateSubstitutions, indexTemplateSubstitutions, request); responses.set( i, BulkItemResponse.success( @@ -153,7 +156,11 @@ protected void doInternalExecute( * @param request The IndexRequest whose source will be validated against the mapping (if it exists) of its index * @return a mapping exception if the source does not match the mappings, otherwise null */ - private Exception validateMappings(Map componentTemplateSubstitutions, IndexRequest request) { + private Exception validateMappings( + Map componentTemplateSubstitutions, + Map indexTemplateSubstitutions, + IndexRequest request + ) { final SourceToParse sourceToParse = new SourceToParse( request.id(), request.source(), @@ -167,7 +174,7 @@ private Exception validateMappings(Map componentTempl Exception mappingValidationException = null; IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(request.index()); try { - if (indexAbstraction != null && componentTemplateSubstitutions.isEmpty()) { + if (indexAbstraction != null && componentTemplateSubstitutions.isEmpty() && indexTemplateSubstitutions.isEmpty()) { /* * In this case the index exists and we don't have any component template overrides. So we can just use withTempIndexService * to do the mapping validation, using all the existing logic for validation. @@ -222,6 +229,12 @@ private Exception validateMappings(Map componentTempl updatedComponentTemplates.putAll(componentTemplateSubstitutions); simulatedMetadata.componentTemplates(updatedComponentTemplates); } + if (indexTemplateSubstitutions.isEmpty() == false) { + Map updatedIndexTemplates = new HashMap<>(); + updatedIndexTemplates.putAll(state.metadata().templatesV2()); + updatedIndexTemplates.putAll(indexTemplateSubstitutions); + simulatedMetadata.indexTemplates(updatedIndexTemplates); + } ClusterState simulatedState = simulatedClusterStateBuilder.metadata(simulatedMetadata).build(); String matchingTemplate = findV2Template(simulatedState.metadata(), request.index(), false); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index abeb3279b7b50..2a2cf6743a877 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1421,44 +1421,24 @@ public static Settings resolveSettings(final List templat * Resolve the given v2 template into a collected {@link Settings} object */ public static Settings resolveSettings(final Metadata metadata, final String templateName) { - return resolveSettings(metadata, templateName, Map.of()); - } - - public static Settings resolveSettings( - final Metadata metadata, - final String templateName, - Map templateSubstitutions - ) { final ComposableIndexTemplate template = metadata.templatesV2().get(templateName); assert template != null : "attempted to resolve settings for a template [" + templateName + "] that did not exist in the cluster state"; if (template == null) { return Settings.EMPTY; } - return resolveSettings(template, metadata.componentTemplates(), templateSubstitutions); + return resolveSettings(template, metadata.componentTemplates()); } /** * Resolve the provided v2 template and component templates into a collected {@link Settings} object */ public static Settings resolveSettings(ComposableIndexTemplate template, Map componentTemplates) { - return resolveSettings(template, componentTemplates, Map.of()); - } - - public static Settings resolveSettings( - ComposableIndexTemplate template, - Map componentTemplates, - Map templateSubstitutions - ) { Objects.requireNonNull(template, "attempted to resolve settings for a null template"); Objects.requireNonNull(componentTemplates, "attempted to resolve settings with null component templates"); - Map combinedComponentTemplates = new HashMap<>(); - combinedComponentTemplates.putAll(componentTemplates); - // We want any substitutions to take precedence: - combinedComponentTemplates.putAll(templateSubstitutions); List componentSettings = template.composedOf() .stream() - .map(combinedComponentTemplates::get) + .map(componentTemplates::get) .filter(Objects::nonNull) .map(ComponentTemplate::template) .map(Template::settings) diff --git a/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java b/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java index 6de15b0046f1b..680860332fe74 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/ingest/RestSimulateIngestAction.java @@ -76,7 +76,8 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC Map sourceMap = XContentHelper.convertToMap(sourceTuple.v2(), false, sourceTuple.v1()).v2(); SimulateBulkRequest bulkRequest = new SimulateBulkRequest( (Map>) sourceMap.remove("pipeline_substitutions"), - (Map>) sourceMap.remove("component_template_substitutions") + (Map>) sourceMap.remove("component_template_substitutions"), + (Map>) sourceMap.remove("index_template_substitutions") ); BytesReference transformedData = convertToBulkRequestXContentBytes(sourceMap); bulkRequest.add( diff --git a/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java index b6b1770e2ed5c..c94e4e46c9ee3 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/SimulateBulkRequestTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.test.ESTestCase; @@ -27,18 +28,27 @@ public class SimulateBulkRequestTests extends ESTestCase { public void testSerialization() throws Exception { - testSerialization(getTestPipelineSubstitutions(), getTestTemplateSubstitutions()); - testSerialization(getTestPipelineSubstitutions(), null); - testSerialization(null, getTestTemplateSubstitutions()); - testSerialization(null, null); - testSerialization(Map.of(), Map.of()); + testSerialization(getTestPipelineSubstitutions(), getTestComponentTemplateSubstitutions(), getTestIndexTemplateSubstitutions()); + testSerialization(getTestPipelineSubstitutions(), null, null); + testSerialization(getTestPipelineSubstitutions(), getTestComponentTemplateSubstitutions(), null); + testSerialization(getTestPipelineSubstitutions(), null, getTestIndexTemplateSubstitutions()); + testSerialization(null, getTestComponentTemplateSubstitutions(), getTestIndexTemplateSubstitutions()); + testSerialization(null, getTestComponentTemplateSubstitutions(), null); + testSerialization(null, null, getTestIndexTemplateSubstitutions()); + testSerialization(null, null, null); + testSerialization(Map.of(), Map.of(), Map.of()); } private void testSerialization( Map> pipelineSubstitutions, - Map> templateSubstitutions + Map> componentTemplateSubstitutions, + Map> indexTemplateSubstitutions ) throws IOException { - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions, templateSubstitutions); + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest( + pipelineSubstitutions, + componentTemplateSubstitutions, + indexTemplateSubstitutions + ); /* * Note: SimulateBulkRequest does not implement equals or hashCode, so we can't test serialization in the usual way for a * Writable @@ -49,7 +59,7 @@ private void testSerialization( @SuppressWarnings({ "unchecked", "rawtypes" }) public void testGetComponentTemplateSubstitutions() throws IOException { - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of()); + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of()); assertThat(simulateBulkRequest.getComponentTemplateSubstitutions(), equalTo(Map.of())); String substituteComponentTemplatesString = """ { @@ -83,7 +93,7 @@ public void testGetComponentTemplateSubstitutions() throws IOException { XContentType.JSON ).v2(); Map> substituteComponentTemplates = (Map>) tempMap; - simulateBulkRequest = new SimulateBulkRequest(Map.of(), substituteComponentTemplates); + simulateBulkRequest = new SimulateBulkRequest(Map.of(), substituteComponentTemplates, Map.of()); Map componentTemplateSubstitutions = simulateBulkRequest.getComponentTemplateSubstitutions(); assertThat(componentTemplateSubstitutions.size(), equalTo(2)); assertThat( @@ -107,8 +117,70 @@ public void testGetComponentTemplateSubstitutions() throws IOException { ); } + public void testGetIndexTemplateSubstitutions() throws IOException { + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), Map.of()); + assertThat(simulateBulkRequest.getIndexTemplateSubstitutions(), equalTo(Map.of())); + String substituteIndexTemplatesString = """ + { + "foo_template": { + "index_patterns": ["foo*"], + "composed_of": ["foo_mapping_template", "foo_settings_template"], + "template": { + "mappings": { + "dynamic": "true", + "properties": { + "foo": { + "type": "keyword" + } + } + }, + "settings": { + "index": { + "default_pipeline": "foo-pipeline" + } + } + } + }, + "bar_template": { + "index_patterns": ["bar*"], + "composed_of": ["bar_mapping_template", "bar_settings_template"] + } + } + """; + + @SuppressWarnings("unchecked") + Map> substituteIndexTemplates = (Map>) (Map) XContentHelper.convertToMap( + new BytesArray(substituteIndexTemplatesString.getBytes(StandardCharsets.UTF_8)), + randomBoolean(), + XContentType.JSON + ).v2(); + simulateBulkRequest = new SimulateBulkRequest(Map.of(), Map.of(), substituteIndexTemplates); + Map indexTemplateSubstitutions = simulateBulkRequest.getIndexTemplateSubstitutions(); + assertThat(indexTemplateSubstitutions.size(), equalTo(2)); + assertThat( + XContentHelper.convertToMap( + XContentHelper.toXContent(indexTemplateSubstitutions.get("foo_template").template(), XContentType.JSON, randomBoolean()), + randomBoolean(), + XContentType.JSON + ).v2(), + equalTo(substituteIndexTemplates.get("foo_template").get("template")) + ); + + assertThat(indexTemplateSubstitutions.get("foo_template").template().settings().size(), equalTo(1)); + assertThat( + indexTemplateSubstitutions.get("foo_template").template().settings().get("index.default_pipeline"), + equalTo("foo-pipeline") + ); + assertNull(indexTemplateSubstitutions.get("bar_template").template()); + assertNull(indexTemplateSubstitutions.get("bar_template").template()); + } + public void testShallowClone() throws IOException { - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(getTestPipelineSubstitutions(), getTestTemplateSubstitutions()); + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest( + getTestPipelineSubstitutions(), + getTestComponentTemplateSubstitutions(), + getTestIndexTemplateSubstitutions() + ); simulateBulkRequest.setRefreshPolicy(randomFrom(WriteRequest.RefreshPolicy.values())); simulateBulkRequest.waitForActiveShards(randomIntBetween(1, 10)); simulateBulkRequest.timeout(randomTimeValue()); @@ -144,7 +216,7 @@ private static Map> getTestPipelineSubstitutions() { ); } - private static Map> getTestTemplateSubstitutions() { + private static Map> getTestComponentTemplateSubstitutions() { return Map.of( "template1", Map.of( @@ -155,4 +227,25 @@ private static Map> getTestTemplateSubstitutions() { Map.of("template", Map.of("mappings", Map.of(), "settings", Map.of())) ); } + + private static Map> getTestIndexTemplateSubstitutions() { + return Map.of( + "template1", + Map.of( + "template", + Map.of( + "index_patterns", + List.of("foo*", "bar*"), + "composed_of", + List.of("template_1", "template_2"), + "mappings", + Map.of("_source", Map.of("enabled", false), "properties", Map.of()), + "settings", + Map.of() + ) + ), + "template2", + Map.of("template", Map.of("index_patterns", List.of("foo*", "bar*"), "mappings", Map.of(), "settings", Map.of())) + ); + } } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java index f4e53912d09a7..71bc31334920e 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java @@ -135,7 +135,7 @@ public void tearDown() throws Exception { public void testIndexData() throws IOException { Task task = mock(Task.class); // unused - BulkRequest bulkRequest = new SimulateBulkRequest(null, null); + BulkRequest bulkRequest = new SimulateBulkRequest(null, null, null); int bulkItemCount = randomIntBetween(0, 200); for (int i = 0; i < bulkItemCount; i++) { Map source = Map.of(randomAlphaOfLength(10), randomAlphaOfLength(5)); @@ -218,7 +218,7 @@ public void testIndexDataWithValidation() throws IOException { * (7) An indexing request to a nonexistent index that matches no templates */ Task task = mock(Task.class); // unused - BulkRequest bulkRequest = new SimulateBulkRequest(null, null); + BulkRequest bulkRequest = new SimulateBulkRequest(null, null, null); int bulkItemCount = randomIntBetween(0, 200); Map indicesMap = new HashMap<>(); Map v1Templates = new HashMap<>(); diff --git a/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java index 332a04e40e43d..3b3f5bdc747b5 100644 --- a/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/SimulateIngestServiceTests.java @@ -65,7 +65,7 @@ public void testGetPipeline() { ingestService.innerUpdatePipelines(ingestMetadata); { // First we make sure that if there are no substitutions that we get our original pipeline back: - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(null, null); + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(null, null, null); SimulateIngestService simulateIngestService = new SimulateIngestService(ingestService, simulateBulkRequest); Pipeline pipeline = simulateIngestService.getPipeline("pipeline1"); assertThat(pipeline.getProcessors(), contains(transformedMatch(Processor::getType, equalTo("processor1")))); @@ -83,7 +83,7 @@ public void testGetPipeline() { ); pipelineSubstitutions.put("pipeline2", newHashMap("processors", List.of(newHashMap("processor3", Collections.emptyMap())))); - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions, null); + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions, null, null); SimulateIngestService simulateIngestService = new SimulateIngestService(ingestService, simulateBulkRequest); Pipeline pipeline1 = simulateIngestService.getPipeline("pipeline1"); assertThat( @@ -103,7 +103,7 @@ public void testGetPipeline() { */ Map> pipelineSubstitutions = new HashMap<>(); pipelineSubstitutions.put("pipeline2", newHashMap("processors", List.of(newHashMap("processor3", Collections.emptyMap())))); - SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions, null); + SimulateBulkRequest simulateBulkRequest = new SimulateBulkRequest(pipelineSubstitutions, null, null); SimulateIngestService simulateIngestService = new SimulateIngestService(ingestService, simulateBulkRequest); Pipeline pipeline1 = simulateIngestService.getPipeline("pipeline1"); assertThat(pipeline1.getProcessors(), contains(transformedMatch(Processor::getType, equalTo("processor1"))));