diff --git a/docs/changelog/112282.yaml b/docs/changelog/112282.yaml new file mode 100644 index 0000000000000..beea119b06aef --- /dev/null +++ b/docs/changelog/112282.yaml @@ -0,0 +1,6 @@ +pr: 112282 +summary: Adds example plugin for custom ingest processor +area: Ingest Node +type: enhancement +issues: + - 111539 diff --git a/docs/plugins/development/creating-classic-plugins.asciidoc b/docs/plugins/development/creating-classic-plugins.asciidoc index f3f62a11f2993..cc03ad51275fa 100644 --- a/docs/plugins/development/creating-classic-plugins.asciidoc +++ b/docs/plugins/development/creating-classic-plugins.asciidoc @@ -32,12 +32,13 @@ for the plugin. If you need other resources, package them into a resources JAR. The {es} repository contains {es-repo}tree/main/plugins/examples[examples of plugins]. Some of these include: * a plugin with {es-repo}tree/main/plugins/examples/custom-settings[custom settings] +* a plugin with a {es-repo}tree/main/plugins/examples/custom-processor[custom ingest processor] * adding {es-repo}tree/main/plugins/examples/rest-handler[custom rest endpoints] * adding a {es-repo}tree/main/plugins/examples/rescore[custom rescorer] * a script {es-repo}tree/main/plugins/examples/script-expert-scoring[implemented in Java] These examples provide the bare bones needed to get started. For more -information about how to write a plugin, we recommend looking at the +information about how to write a plugin, we recommend looking at the {es-repo}tree/main/plugins/[source code of existing plugins] for inspiration. [discrete] @@ -88,4 +89,4 @@ for more information. [[plugin-descriptor-file-classic]] ==== The plugin descriptor file for classic plugins -include::plugin-descriptor-file.asciidoc[] \ No newline at end of file +include::plugin-descriptor-file.asciidoc[] diff --git a/plugins/examples/custom-processor/build.gradle b/plugins/examples/custom-processor/build.gradle new file mode 100644 index 0000000000000..69da64d8ebe86 --- /dev/null +++ b/plugins/examples/custom-processor/build.gradle @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +apply plugin: 'elasticsearch.esplugin' +apply plugin: 'elasticsearch.yaml-rest-test' + +esplugin { + name 'custom-processor' + description 'An example plugin showing how to register a custom ingest processor' + classname 'org.elasticsearch.example.customprocessor.ExampleProcessorPlugin' + licenseFile rootProject.file('SSPL-1.0+ELASTIC-LICENSE-2.0.txt') + noticeFile rootProject.file('NOTICE.txt') +} + +dependencies { + yamlRestTestRuntimeOnly "org.apache.logging.log4j:log4j-core:${log4jVersion}" +} diff --git a/plugins/examples/custom-processor/src/main/java/org/elasticsearch/example/customprocessor/ExampleProcessorPlugin.java b/plugins/examples/custom-processor/src/main/java/org/elasticsearch/example/customprocessor/ExampleProcessorPlugin.java new file mode 100644 index 0000000000000..1ba145a92ca7d --- /dev/null +++ b/plugins/examples/custom-processor/src/main/java/org/elasticsearch/example/customprocessor/ExampleProcessorPlugin.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.example.customprocessor; + +import org.elasticsearch.ingest.Processor; +import org.elasticsearch.plugins.IngestPlugin; +import org.elasticsearch.plugins.Plugin; + +import java.util.Map; + +public class ExampleProcessorPlugin extends Plugin implements IngestPlugin { + + @Override + public Map getProcessors(Processor.Parameters parameters) { + return Map.of(ExampleRepeatProcessor.TYPE, new ExampleRepeatProcessor.Factory()); + } +} diff --git a/plugins/examples/custom-processor/src/main/java/org/elasticsearch/example/customprocessor/ExampleRepeatProcessor.java b/plugins/examples/custom-processor/src/main/java/org/elasticsearch/example/customprocessor/ExampleRepeatProcessor.java new file mode 100644 index 0000000000000..f0f942459281a --- /dev/null +++ b/plugins/examples/custom-processor/src/main/java/org/elasticsearch/example/customprocessor/ExampleRepeatProcessor.java @@ -0,0 +1,53 @@ +package org.elasticsearch.example.customprocessor; + +import org.elasticsearch.ingest.AbstractProcessor; +import org.elasticsearch.ingest.ConfigurationUtils; +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.Processor; + +import java.util.Map; + +/** + * Example of adding an ingest processor with a plugin. + */ +public class ExampleRepeatProcessor extends AbstractProcessor { + public static final String TYPE = "repeat"; + public static final String FIELD_KEY_NAME = "field"; + + private final String field; + + ExampleRepeatProcessor(String tag, String description, String field) { + super(tag, description); + this.field = field; + } + + @Override + public IngestDocument execute(IngestDocument document) { + Object val = document.getFieldValue(field, Object.class, true); + + if (val instanceof String string) { + String repeated = string.concat(string); + document.setFieldValue(field, repeated); + } + return document; + } + + @Override + public String getType() { + return TYPE; + } + + public static class Factory implements Processor.Factory { + + @Override + public ExampleRepeatProcessor create( + Map registry, + String tag, + String description, + Map config + ) { + String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, FIELD_KEY_NAME); + return new ExampleRepeatProcessor(tag, description, field); + } + } +} diff --git a/plugins/examples/custom-processor/src/yamlRestTest/java/org/elasticsearch/example/customprocessor/ExampleProcessorClientYamlTestSuiteIT.java b/plugins/examples/custom-processor/src/yamlRestTest/java/org/elasticsearch/example/customprocessor/ExampleProcessorClientYamlTestSuiteIT.java new file mode 100644 index 0000000000000..ac08df358fe5e --- /dev/null +++ b/plugins/examples/custom-processor/src/yamlRestTest/java/org/elasticsearch/example/customprocessor/ExampleProcessorClientYamlTestSuiteIT.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.example.customprocessor; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; + +/** + * {@link ExampleProcessorClientYamlTestSuiteIT} executes the plugin's REST API integration tests. + *

+ * The tests can be executed using the command: ./gradlew :custom-processor:yamlRestTest + *

+ * This class extends {@link ESClientYamlSuiteTestCase}, which takes care of parsing the YAML files + * located in the src/yamlRestTest/resources/rest-api-spec/test/ directory and validates them against the + * custom REST API definition files located in src/yamlRestTest/resources/rest-api-spec/api/. + *

+ * Once validated, {@link ESClientYamlSuiteTestCase} executes the REST tests against a single node + * integration cluster which has the plugin already installed by the Gradle build script. + *

+ */ +public class ExampleProcessorClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + public ExampleProcessorClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + // The test executes all the test candidates by default + // see ESClientYamlSuiteTestCase.REST_TESTS_SUITE + return ESClientYamlSuiteTestCase.createParameters(); + } +} diff --git a/plugins/examples/custom-processor/src/yamlRestTest/resources/rest-api-spec/test/customprocessor/10_basic.yml b/plugins/examples/custom-processor/src/yamlRestTest/resources/rest-api-spec/test/customprocessor/10_basic.yml new file mode 100644 index 0000000000000..40f5835fe9760 --- /dev/null +++ b/plugins/examples/custom-processor/src/yamlRestTest/resources/rest-api-spec/test/customprocessor/10_basic.yml @@ -0,0 +1,15 @@ +"Custom processor is present": + - do: + ingest.put_pipeline: + id: pipeline1 + body: > + { + "processors": [ + { + "repeat" : { + "field": "test" + } + } + ] + } + - match: { acknowledged: true } diff --git a/plugins/examples/custom-processor/src/yamlRestTest/resources/rest-api-spec/test/customprocessor/20_process_document.yml b/plugins/examples/custom-processor/src/yamlRestTest/resources/rest-api-spec/test/customprocessor/20_process_document.yml new file mode 100644 index 0000000000000..7e8bc2e0a2d78 --- /dev/null +++ b/plugins/examples/custom-processor/src/yamlRestTest/resources/rest-api-spec/test/customprocessor/20_process_document.yml @@ -0,0 +1,59 @@ +setup: + - do: + ingest.put_pipeline: + id: pipeline1 + body: > + { + "processors": [ + { + "repeat" : { + "field": "to_repeat" + } + } + ] + } +--- +teardown: + - do: + ingest.delete_pipeline: + id: pipeline1 + ignore: 404 + + - do: + indices.delete: + index: index1 + ignore: 404 +--- +"Process document": + # index a document with field to be processed + - do: + index: + id: doc1 + index: index1 + pipeline: pipeline1 + body: { to_repeat: "foo" } + - match: { result: "created" } + + # validate document is processed + - do: + get: + index: index1 + id: doc1 + - match: { _source: { to_repeat: "foofoo" } } +--- +"Does not process document without field": + # index a document without field to be processed + - do: + index: + id: doc1 + index: index1 + pipeline: pipeline1 + body: { field1: "foo" } + - match: { result: "created" } + + # validate document is not processed + - do: + get: + index: index1 + id: doc1 + - match: { _source: { field1: "foo" } }