Add option for Append Processor to skip/allow empty values

docs/reference/ingest/processors/append.asciidoc

@@ -19,6 +19,8 @@ Accepts a single value or an array of values.
 | `value`   | yes       | -        | The value to be appended. Supports <<template-snippets,template snippets>>.
 | `allow_duplicates` | no | true  | If `false`, the processor does not append
 values already present in the field.
+| `allow_empty_values` | no | true  | If `false`, the processor does not append
+values that resolve to empty strings.
 | `media_type` | no | `application/json` | The media type for encoding `value`. Applies only when `value` is a
 <<template-snippets,template snippet>>. Must be one of `application/json`, `text/plain`, or
 `application/x-www-form-urlencoded`.

modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java

@@ -31,12 +31,21 @@ public final class AppendProcessor extends AbstractProcessor {
     private final TemplateScript.Factory field;
     private final ValueSource value;
     private final boolean allowDuplicates;
+    private final boolean allowEmptyValues;
 
-    AppendProcessor(String tag, String description, TemplateScript.Factory field, ValueSource value, boolean allowDuplicates) {
+    AppendProcessor(
+        String tag,
+        String description,
+        TemplateScript.Factory field,
+        ValueSource value,
+        boolean allowDuplicates,
+        boolean allowEmptyValues
+    ) {
         super(tag, description);
         this.field = field;
         this.value = value;
         this.allowDuplicates = allowDuplicates;
+        this.allowEmptyValues = allowEmptyValues;
     }
 
     public TemplateScript.Factory getField() {
@@ -50,7 +59,7 @@ public ValueSource getValue() {
     @Override
     public IngestDocument execute(IngestDocument document) throws Exception {
         String path = document.renderTemplate(field);
-        document.appendFieldValue(path, value, allowDuplicates);
+        document.appendFieldValue(path, value, allowDuplicates, allowEmptyValues);
         return document;
     }
 
@@ -77,14 +86,16 @@ public AppendProcessor create(
             String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field");
             Object value = ConfigurationUtils.readObject(TYPE, processorTag, config, "value");
             boolean allowDuplicates = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "allow_duplicates", true);
+            boolean allowEmptyValues = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "allow_empty_values", true);
             TemplateScript.Factory compiledTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag, "field", field, scriptService);
             String mediaType = ConfigurationUtils.readMediaTypeProperty(TYPE, processorTag, config, "media_type", "application/json");
             return new AppendProcessor(
                 processorTag,
                 description,
                 compiledTemplate,
                 ValueSource.wrap(value, scriptService, Map.of(Script.CONTENT_TYPE_OPTION, mediaType)),
-                allowDuplicates
+                allowDuplicates,
+                allowEmptyValues
             );
         }
     }

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/AppendProcessorTests.java

@@ -50,13 +50,13 @@ public void testAppendValuesToExistingList() throws Exception {
         if (randomBoolean()) {
             Object value = scalar.randomValue();
             values.add(value);
-            appendProcessor = createAppendProcessor(field, value, true);
+            appendProcessor = createAppendProcessor(field, value, true, true);
         } else {
             int valuesSize = randomIntBetween(0, 10);
             for (int i = 0; i < valuesSize; i++) {
                 values.add(scalar.randomValue());
             }
-            appendProcessor = createAppendProcessor(field, values, true);
+            appendProcessor = createAppendProcessor(field, values, true, true);
         }
         appendProcessor.execute(ingestDocument);
         Object fieldValue = ingestDocument.getFieldValue(field, Object.class);
@@ -79,13 +79,13 @@ public void testAppendValuesToNonExistingList() throws Exception {
         if (randomBoolean()) {
             Object value = scalar.randomValue();
             values.add(value);
-            appendProcessor = createAppendProcessor(field, value, true);
+            appendProcessor = createAppendProcessor(field, value, true, true);
         } else {
             int valuesSize = randomIntBetween(0, 10);
             for (int i = 0; i < valuesSize; i++) {
                 values.add(scalar.randomValue());
             }
-            appendProcessor = createAppendProcessor(field, values, true);
+            appendProcessor = createAppendProcessor(field, values, true, true);
         }
         appendProcessor.execute(ingestDocument);
         List<?> list = ingestDocument.getFieldValue(field, List.class);
@@ -103,13 +103,13 @@ public void testConvertScalarToList() throws Exception {
         if (randomBoolean()) {
             Object value = scalar.randomValue();
             values.add(value);
-            appendProcessor = createAppendProcessor(field, value, true);
+            appendProcessor = createAppendProcessor(field, value, true, true);
         } else {
             int valuesSize = randomIntBetween(0, 10);
             for (int i = 0; i < valuesSize; i++) {
                 values.add(scalar.randomValue());
             }
-            appendProcessor = createAppendProcessor(field, values, true);
+            appendProcessor = createAppendProcessor(field, values, true, true);
         }
         appendProcessor.execute(ingestDocument);
         List<?> fieldValue = ingestDocument.getFieldValue(field, List.class);
@@ -127,7 +127,7 @@ public void testAppendingDuplicateValueToScalarDoesNotModifyDocument() throws Ex
 
         List<Object> valuesToAppend = new ArrayList<>();
         valuesToAppend.add(originalValue);
-        Processor appendProcessor = createAppendProcessor(field, valuesToAppend, false);
+        Processor appendProcessor = createAppendProcessor(field, valuesToAppend, false, true);
         appendProcessor.execute(ingestDocument);
         Object fieldValue = ingestDocument.getFieldValue(field, Object.class);
         assertThat(fieldValue, not(instanceOf(List.class)));
@@ -142,7 +142,7 @@ public void testAppendingUniqueValueToScalar() throws Exception {
         List<Object> valuesToAppend = new ArrayList<>();
         String newValue = randomValueOtherThan(originalValue, () -> randomAlphaOfLengthBetween(1, 10));
         valuesToAppend.add(newValue);
-        Processor appendProcessor = createAppendProcessor(field, valuesToAppend, false);
+        Processor appendProcessor = createAppendProcessor(field, valuesToAppend, false, true);
         appendProcessor.execute(ingestDocument);
         List<?> list = ingestDocument.getFieldValue(field, List.class);
         assertThat(list.size(), equalTo(2));
@@ -171,20 +171,140 @@ public void testAppendingToListWithDuplicatesDisallowed() throws Exception {
         Collections.sort(valuesToAppend);
 
         // attempt to append both new and existing values
-        Processor appendProcessor = createAppendProcessor(originalField, valuesToAppend, false);
+        Processor appendProcessor = createAppendProcessor(originalField, valuesToAppend, false, true);
         appendProcessor.execute(ingestDocument);
         List<?> fieldValue = ingestDocument.getFieldValue(originalField, List.class);
         assertThat(fieldValue, sameInstance(list));
         assertThat(fieldValue, containsInAnyOrder(expectedValues.toArray()));
     }
 
-    private static Processor createAppendProcessor(String fieldName, Object fieldValue, boolean allowDuplicates) {
+    public void testAppendingToListWithNoEmptyValuesAndEmptyValuesDisallowed() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        Scalar scalar = randomFrom(Scalar.values());
+        List<Object> list = new ArrayList<>();
+        int size = randomIntBetween(0, 10);
+        for (int i = 0; i < size; i++) {
+            list.add(scalar.randomValue());
+        }
+        List<Object> checkList = new ArrayList<>(list);
+        String field = RandomDocumentPicks.addRandomField(random(), ingestDocument, list);
+        List<Object> values = new ArrayList<>();
+        Processor appendProcessor;
+        if (randomBoolean()) {
+            Object value = scalar.randomValue();
+            values.add(value);
+            appendProcessor = createAppendProcessor(field, value, true, false);
+        } else {
+            int valuesSize = randomIntBetween(0, 10);
+            for (int i = 0; i < valuesSize; i++) {
+                values.add(scalar.randomValue());
+            }
+            appendProcessor = createAppendProcessor(field, values, true, false);
+        }
+        appendProcessor.execute(ingestDocument);
+        Object fieldValue = ingestDocument.getFieldValue(field, Object.class);
+        assertThat(fieldValue, sameInstance(list));
+        assertThat(list.size(), equalTo(size + values.size()));
+        for (int i = 0; i < size; i++) {
+            assertThat(list.get(i), equalTo(checkList.get(i)));
+        }
+        for (int i = size; i < size + values.size(); i++) {
+            assertThat(list.get(i), equalTo(values.get(i - size)));
+        }
+    }
+
+    public void testAppendingToListEmptyStringAndEmptyValuesDisallowed() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        Scalar scalar = Scalar.STRING;
+        List<Object> list = new ArrayList<>();
+        int size = randomIntBetween(0, 10);
+        for (int i = 0; i < size; i++) {
+            list.add(scalar.randomValue());
+        }
+        List<Object> checkList = new ArrayList<>(list);
+        String field = RandomDocumentPicks.addRandomField(random(), ingestDocument, list);
+        List<Object> values = new ArrayList<>();
+        Processor appendProcessor;
+        if (randomBoolean()) {
+            Object value;
+            if (randomBoolean()) {
+                value = "";
+            } else {
+                value = scalar.randomValue();
+                values.add(value);
+            }
+            appendProcessor = createAppendProcessor(field, value, true, false);
+        } else {
+            int valuesSize = randomIntBetween(0, 10);
+            List<Object> allValues = new ArrayList<>();
+            for (int i = 0; i < valuesSize; i++) {
+                Object value;
+                if (randomBoolean()) {
+                    value = "";
+                } else {
+                    value = scalar.randomValue();
+                    values.add(value);
+                }
+                allValues.add(value);
+            }
+            appendProcessor = createAppendProcessor(field, allValues, true, false);
+        }
+        appendProcessor.execute(ingestDocument);
+        Object fieldValue = ingestDocument.getFieldValue(field, Object.class);
+        assertThat(fieldValue, sameInstance(list));
+        assertThat(list.size(), equalTo(size + values.size()));
+        for (int i = 0; i < size; i++) {
+            assertThat(list.get(i), equalTo(checkList.get(i)));
+        }
+        for (int i = size; i < size + values.size(); i++) {
+            assertThat(list.get(i), equalTo(values.get(i - size)));
+        }
+    }
+
+    public void testAppendingToNonExistingListEmptyStringAndEmpyValuesDisallowed() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
+        String field = RandomDocumentPicks.randomFieldName(random());
+        Scalar scalar = Scalar.STRING;
+        List<Object> values = new ArrayList<>();
+        Processor appendProcessor;
+        if (randomBoolean()) {
+            Object value;
+            if (randomBoolean()) {
+                value = "";
+            } else {
+                value = scalar.randomValue();
+                values.add(value);
+            }
+            appendProcessor = createAppendProcessor(field, value, true, false);
+        } else {
+            List<Object> allValues = new ArrayList<>();
+            int valuesSize = randomIntBetween(0, 10);
+            for (int i = 0; i < valuesSize; i++) {
+                Object value;
+                if (randomBoolean()) {
+                    value = "";
+                } else {
+                    value = scalar.randomValue();
+                    values.add(value);
+                }
+                allValues.add(value);
+            }
+            appendProcessor = createAppendProcessor(field, allValues, true, false);
+        }
+        appendProcessor.execute(ingestDocument);
+        List<?> list = ingestDocument.getFieldValue(field, List.class);
+        assertThat(list, not(sameInstance(values)));
+        assertThat(list, equalTo(values));
+    }
+
+    private static Processor createAppendProcessor(String fieldName, Object fieldValue, boolean allowDuplicates, boolean allowEmptyValues) {
         return new AppendProcessor(
             randomAlphaOfLength(10),
             null,
             new TestTemplateService.MockTemplateScript.Factory(fieldName),
             ValueSource.wrap(fieldValue, TestTemplateService.instance()),
-            allowDuplicates
+            allowDuplicates,
+            allowEmptyValues
         );
     }
 

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java

@@ -199,7 +199,7 @@ public void testModifyFieldsOutsideArray() {
             new CompoundProcessor(
                 false,
                 List.of(new UppercaseProcessor("_tag_upper", null, "_ingest._value", false, "_ingest._value")),
-                List.of(new AppendProcessor("_tag", null, template, (model) -> (List.of("added")), true))
+                List.of(new AppendProcessor("_tag", null, template, (model) -> (List.of("added")), true, true))
             ),
             false
         );