Add option for Append Processor to skip/allow empty values

docs/changelog/105718.yaml

@@ -0,0 +1,6 @@
+pr: 105718
+summary: Add option for Append Processor to skip/allow empty values
+area: Ingest Node
+type: enhancement
+issues:
+ - 104813

docs/reference/enrich-processor/append-processor.md

@@ -15,15 +15,22 @@ $$$append-options$$$
 | --- | --- | --- | --- |
 | `field` | yes | - | The field to be appended to. Supports [template snippets](docs-content://manage-data/ingest/transform-enrich/ingest-pipelines.md#template-snippets). |
 | `value` | yes* | - | The value to be appended. Supports [template snippets](docs-content://manage-data/ingest/transform-enrich/ingest-pipelines.md#template-snippets). May specify only one of `value` or `copy_from`. |
-| `copy_from` {applies_to}`stack: ga 9.2.0` | no | - | The origin field which will be appended to `field`, cannot set `value` simultaneously. |
+| `copy_from` {applies_to}`stack: ga 9.2` | no | - | The origin field which will be appended to `field`, cannot set `value` simultaneously. |
 | `allow_duplicates` | no | true | If `false`, the processor does not appendvalues already present in the field. |
+| `ignore_empty_values` {applies_to}`stack: ga 9.2` | no | false | If `true`, the processor does not append values that resolve to `null` or an empty string.
 | `media_type` | no | `application/json` | The media type for encoding `value`. Applies only when `value` is a [template snippet](docs-content://manage-data/ingest/transform-enrich/ingest-pipelines.md#template-snippets). Must be one of `application/json`, `text/plain`, or`application/x-www-form-urlencoded`. |
 | `description` | no | - | Description of the processor. Useful for describing the purpose of the processor or its configuration. |
 | `if` | no | - | Conditionally execute the processor. See [Conditionally run a processor](docs-content://manage-data/ingest/transform-enrich/ingest-pipelines.md#conditionally-run-processor). |
 | `ignore_failure` | no | `false` | Ignore failures for the processor. See [Handling pipeline failures](docs-content://manage-data/ingest/transform-enrich/ingest-pipelines.md#handling-pipeline-failures). |
 | `on_failure` | no | - | Handle failures for the processor. See [Handling pipeline failures](docs-content://manage-data/ingest/transform-enrich/ingest-pipelines.md#handling-pipeline-failures). |
 | `tag` | no | - | Identifier for the processor. Useful for debugging and metrics. |
 
+## Examples [append-processor-examples]
+
+### Simple example [append-processor-simple-example]
+
+Here is an `append` processor definition that adds the string `"production"` as well as the values of the `app` and `owner` fields to the `tags` field:
+
 ```js
 {
   "append": {
@@ -33,3 +40,21 @@ $$$append-options$$$
 }
 ```
 
+### Example using `allow_duplicates` and `ignore_empty_values` [append-processor-example-using-allow-duplicates-and-ignore-empty-values]
+
+```{applies_to}
+stack: ga 9.2
+```
+
+By using `allow_duplicates` and `ignore_empty_values`, it is possible to only append the `host.name` to the `related.hosts` if the `host.name` is not empty and if the value is not already present in `related.hosts`:
+
+```js
+{
+  "append": {
+    "field": "related.hosts",
+    "copy_from": "host.name",
+    "allow_duplicates": false,
+    "ignore_empty_values": true
+  }
+}
+```

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java

@@ -36,20 +36,23 @@ public final class AppendProcessor extends AbstractProcessor {
     private final ValueSource value;
     private final String copyFrom;
     private final boolean allowDuplicates;
+    private final boolean ignoreEmptyValues;
 
     AppendProcessor(
         String tag,
         String description,
         TemplateScript.Factory field,
         ValueSource value,
         String copyFrom,
-        boolean allowDuplicates
+        boolean allowDuplicates,
+        boolean ignoreEmptyValues
     ) {
         super(tag, description);
         this.field = field;
         this.value = value;
         this.copyFrom = copyFrom;
         this.allowDuplicates = allowDuplicates;
+        this.ignoreEmptyValues = ignoreEmptyValues;
     }
 
     public TemplateScript.Factory getField() {
@@ -68,10 +71,10 @@ public String getCopyFrom() {
     public IngestDocument execute(IngestDocument document) throws Exception {
         String path = document.renderTemplate(field);
         if (copyFrom != null) {
-            Object fieldValue = document.getFieldValue(copyFrom, Object.class);
-            document.appendFieldValue(path, IngestDocument.deepCopy(fieldValue), allowDuplicates);
+            Object fieldValue = document.getFieldValue(copyFrom, Object.class, ignoreEmptyValues);
+            document.appendFieldValue(path, IngestDocument.deepCopy(fieldValue), allowDuplicates, ignoreEmptyValues);
         } else {
-            document.appendFieldValue(path, value, allowDuplicates);
+            document.appendFieldValue(path, value, allowDuplicates, ignoreEmptyValues);
         }
         return document;
     }
@@ -116,9 +119,17 @@ public AppendProcessor create(
                 }
             }
             boolean allowDuplicates = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "allow_duplicates", true);
+            boolean ignoreEmptyValues = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_empty_values", false);
             TemplateScript.Factory compiledTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag, "field", field, scriptService);
-
-            return new AppendProcessor(processorTag, description, compiledTemplate, valueSource, copyFrom, allowDuplicates);
+            return new AppendProcessor(
+                processorTag,
+                description,
+                compiledTemplate,
+                valueSource,
+                copyFrom,
+                allowDuplicates,
+                ignoreEmptyValues
+            );
         }
     }
 }