Add lookup() option to print unknown values. (#212)

Author: blackwinter

README.md

@@ -562,6 +562,17 @@ Options:
 
 - `__default`: Default value to use for unknown values. (Default: Old value)
 - `delete`: Whether to delete unknown values. (Default: `false`)
+- `print_unknown`: Whether to print unknown values. (Default: `false`)
+
+Additional options when printing unknown values:
+
+- `compression` (file output only): Compression mode. (Default: `auto`)
+- `destination`: Destination to write unknown values to; may include [format directives](https://docs.oracle.com/javase/8/docs/api/java/util/Formatter.html#syntax) for counter and record ID (in that order). (Default: `stdout`)
+- `encoding` (file output only): Encoding used by the underlying writer. (Default: `UTF-8`)
+- `footer`: Footer which is written at the end of the output. (Default: `\n`)
+- `header`: Header which is written at the beginning of the output. (Default: Empty string)
+- `id`: Field name which contains the record ID; if found, will be available for inclusion in `destination`. (Default: `_id`)
+- `separator`: Separator which is written after the unknown value. (Default: `\n`)
 
 ```perl
 lookup("<sourceField>"[, <mapName>][, <options>...])
@@ -586,6 +597,9 @@ lookup("path.to.field", "file-map")
 
 # with default value
 lookup("path.to.field", "map-name", __default: "NA")
+
+# with printing unknown values to a file
+lookup("path.to.field", "map-name", print_unknown: "true", destination: "unknown.txt")
 ```
 
 ##### `prepend`

metafix/src/main/java/org/metafacture/metafix/FixMethod.java

@@ -34,6 +34,7 @@
 import java.util.Map;
 import java.util.Random;
 import java.util.concurrent.atomic.LongAdder;
+import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.function.UnaryOperator;
@@ -241,7 +242,6 @@ private boolean literalString(final String s) {
 
         @Override
         public void apply(final Metafix metafix, final Record record, final List<String> params, final Map<String, String> options) {
-            final String destination = options.getOrDefault("destination", ObjectWriter.STDOUT);
             final Value idValue = record.get(options.getOrDefault("id", StandardEventNames.ID));
 
             final boolean internal = getBoolean(options, "internal");
@@ -250,33 +250,29 @@ public void apply(final Metafix metafix, final Record record, final List<String>
             final LongAdder counter = scopedCounter.computeIfAbsent(metafix, k -> new LongAdder());
             counter.increment();
 
-            final String id = Value.isNull(idValue) ? "" : idValue.toString();
-            final String prefix = params.isEmpty() ? "" : String.format(params.get(0), counter.sum(), id);
-            final ObjectWriter<String> writer = new ObjectWriter<>(String.format(destination, counter.sum(), id));
+            final UnaryOperator<String> formatter = s -> String.format(s,
+                    counter.sum(), Value.isNull(idValue) ? "" : idValue.toString());
 
-            withOption(options, "compression", writer::setCompression);
-            withOption(options, "encoding", writer::setEncoding);
-            withOption(options, "footer", writer::setFooter);
-            withOption(options, "header", writer::setHeader);
+            final String prefix = params.isEmpty() ? "" : formatter.apply(params.get(0));
 
-            if (internal) {
-                if (pretty) {
-                    record.forEach((f, v) -> writer.process(prefix + f + "=" + v));
+            withWriter(options, formatter, w -> {
+                if (internal) {
+                    if (pretty) {
+                        record.forEach((f, v) -> w.process(prefix + f + "=" + v));
+                    }
+                    else {
+                        w.process(prefix + record);
+                    }
                 }
                 else {
-                    writer.process(prefix + record);
-                }
-            }
-            else {
-                try {
-                    writer.process(prefix + record.toJson(pretty));
-                }
-                catch (final IOException e) {
-                    // Log a warning? Print string representation instead?
+                    try {
+                        w.process(prefix + record.toJson(pretty));
+                    }
+                    catch (final IOException e) {
+                        // Log a warning? Print string representation instead?
+                    }
                 }
-            }
-
-            writer.closeStream();
+            });
         }
     },
     random {
@@ -501,10 +497,29 @@ public void apply(final Metafix metafix, final Record record, final List<String>
             }
 
             final String defaultValue = map.get(Maps.DEFAULT_MAP_KEY); // TODO: Catmandu uses 'default'
-            record.transform(params.get(0), oldValue -> {
-                final String newValue = map.getOrDefault(oldValue, defaultValue);
-                return newValue != null ? newValue : getBoolean(options, "delete") ? null : oldValue;
+            final boolean delete = getBoolean(options, "delete");
+            final boolean printUnknown = getBoolean(options, "print_unknown");
+
+            final Consumer<ObjectWriter<String>> consumer = w -> record.transform(params.get(0), oldValue -> {
+                final String newValue = map.get(oldValue);
+                if (newValue != null) {
+                    return newValue;
+                }
+                else {
+                    if (w != null) {
+                        w.process(oldValue);
+                    }
+
+                    return defaultValue != null ? defaultValue : delete ? null : oldValue;
+                }
             });
+
+            if (printUnknown) {
+                withWriter(options, null, consumer);
+            }
+            else {
+                consumer.accept(null);
+            }
         }
     },
     prepend {

metafix/src/main/java/org/metafacture/metafix/api/FixFunction.java

@@ -16,6 +16,7 @@
 
 package org.metafacture.metafix.api;
 
+import org.metafacture.io.ObjectWriter;
 import org.metafacture.metafix.Metafix;
 import org.metafacture.metafix.Record;
 import org.metafacture.metafix.Value;
@@ -26,6 +27,7 @@
 import java.util.Set;
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
+import java.util.function.UnaryOperator;
 import java.util.stream.Stream;
 
 @FunctionalInterface
@@ -43,6 +45,24 @@ default <T> void withOption(final Map<String, String> options, final String key,
         }
     }
 
+    default void withWriter(final Map<String, String> options, final UnaryOperator<String> operator, final Consumer<ObjectWriter<String>> consumer) {
+        final String destination = options.getOrDefault("destination", ObjectWriter.STDOUT);
+        final ObjectWriter<String> writer = new ObjectWriter<>(operator != null ? operator.apply(destination) : destination);
+
+        withOption(options, "compression", writer::setCompression);
+        withOption(options, "encoding", writer::setEncoding);
+        withOption(options, "footer", writer::setFooter);
+        withOption(options, "header", writer::setHeader);
+        withOption(options, "separator", writer::setSeparator);
+
+        try {
+            consumer.accept(writer);
+        }
+        finally {
+            writer.closeStream();
+        }
+    }
+
     default boolean getBoolean(final Map<String, String> options, final String key) {
         return Boolean.parseBoolean(options.get(key));
     }

metafix/src/test/java/org/metafacture/metafix/MetafixLookupTest.java

@@ -24,6 +24,7 @@
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 
+import java.io.IOException;
 import java.util.Arrays;
 
 /**
@@ -846,6 +847,96 @@ public void shouldFailLookupInUnknownExternalMap() {
         );
     }
 
+    private void shouldPrintUnknown(final String args, final String defaultValue, final String expected) {
+        MetafixTestHelpers.assertStdout(expected, () ->
+            MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(
+                    LOOKUP + " Aloha: Alohaeha, 'Moin': 'Moin zäme'" + args + ", print_unknown: 'true')"
+                ),
+                i -> {
+                    i.startRecord("rec1");
+                    i.literal("title", "Aloha");
+                    i.literal("title", "Moin");
+                    i.literal("title", "Hey");
+                    i.literal("title", "you");
+                    i.literal("title", "there");
+                    i.endRecord();
+
+                    i.startRecord("rec2");
+                    i.literal("title", "Aloha");
+                    i.literal("title", "you");
+                    i.literal("title", "too");
+                    i.endRecord();
+                },
+                (o, f) -> {
+                    final boolean delete = "__delete".equals(defaultValue);
+
+                    o.get().startRecord("rec1");
+                    o.get().literal("title", "Alohaeha");
+                    o.get().literal("title", "Moin zäme");
+
+                    if (defaultValue == null) {
+                        o.get().literal("title", "Hey");
+                        o.get().literal("title", "you");
+                        o.get().literal("title", "there");
+                    }
+                    else if (!delete) {
+                        f.apply(3).literal("title", defaultValue);
+                    }
+
+                    o.get().endRecord();
+
+                    o.get().startRecord("rec2");
+                    o.get().literal("title", "Alohaeha");
+
+                    if (defaultValue == null) {
+                        o.get().literal("title", "you");
+                        o.get().literal("title", "too");
+                    }
+                    else if (!delete) {
+                        f.apply(2).literal("title", defaultValue);
+                    }
+
+                    o.get().endRecord();
+                }
+            )
+        );
+    }
+
+    @Test
+    public void shouldPrintUnknown() {
+        shouldPrintUnknown("", null, "Hey\nyou\nthere\nyou\ntoo\n");
+    }
+
+    @Test
+    public void shouldPrintUnknownWithDefault() {
+        shouldPrintUnknown(", __default: Tach", "Tach", "Hey\nyou\nthere\nyou\ntoo\n");
+    }
+
+    @Test
+    public void shouldPrintUnknownWithDelete() {
+        shouldPrintUnknown(", delete: 'true'", "__delete", "Hey\nyou\nthere\nyou\ntoo\n");
+    }
+
+    @Test
+    public void shouldPrintUnknownWithHeader() {
+        shouldPrintUnknown(", header: '<%d:%s>'", null, "<%d:%s>Hey\nyou\nthere\n<%d:%s>you\ntoo\n");
+    }
+
+    @Test
+    public void shouldPrintUnknownWithFooter() {
+        shouldPrintUnknown(", footer: '<%d:%s>'", null, "Hey\nyou\nthere<%d:%s>you\ntoo<%d:%s>");
+    }
+
+    @Test
+    public void shouldPrintUnknownWithSeparator() {
+        shouldPrintUnknown(", separator: '<%d:%s>'", null, "Hey<%d:%s>you<%d:%s>there\nyou<%d:%s>too\n");
+    }
+
+    @Test
+    public void shouldPrintUnknownToFile() throws IOException {
+        MetafixTestHelpers.assertTempFile("you\ntoo\n", p -> shouldPrintUnknown(", destination: '" + p + "'", null, ""));
+    }
+
     private void assertMap(final String... fixDef) {
         MetafixTestHelpers.assertFix(streamReceiver, Arrays.asList(fixDef),
             i -> {