Implement non-normalized PICA+ serialization

This should be backwards compatible as the default is still the normalized
serialization.

- add LineRecorder to split non-normalized serialized PICA+ records
- add testing of LineRecorder
- add flux example
- add flux command

See #296.

Author: dr0i

metafacture-biblio/src/main/java/org/metafacture/biblio/pica/PicaConstants.java

@@ -1,5 +1,4 @@
-/*
- * Copyright 2016 Christoph Böhme
+/* Copyright 2016,2019 Christoph Böhme and hbz
  *
  * Licensed under the Apache License, Version 2.0 the "License";
  * you may not use this file except in compliance with the License.
@@ -13,20 +12,38 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.metafacture.biblio.pica;
 
 /**
- * Useful constants for PICA+
+ * Useful constants for PICA+.
+ * PICA+ comes with two possible serializations:
+ * a normalized one and a non-normalized.
  *
  * @author Christoph Böhme
+ * @author Pascal Christoph (dr0i)
  *
  */
-final class PicaConstants {
 
-    public static final char RECORD_MARKER = '\u001d';
-    public static final char FIELD_MARKER = '\u001e';
-    public static final char SUBFIELD_MARKER = '\u001f';
-    public static final char FIELD_END_MARKER = '\n';
+package org.metafacture.biblio.pica;
+
+final class PicaConstants{
+    public static char RECORD_MARKER = '\u001d';
+    public static char FIELD_MARKER = '\u001e';
+    public static char SUBFIELD_MARKER = '\u001f';
+    public static char FIELD_END_MARKER = '\n';
+
+    public static void setNormalizedSerialization() {
+        RECORD_MARKER = '\u001d';
+        FIELD_MARKER = '\u001e';
+        SUBFIELD_MARKER = '\u001f';
+        FIELD_END_MARKER = '\n';
+    }
+
+    public static void setNonNormalizedSerialization() {
+        RECORD_MARKER = '\n';
+        FIELD_MARKER = '\n'; //this is a dummy
+        SUBFIELD_MARKER = '$';
+        FIELD_END_MARKER = '\n';
+    }
 
     private PicaConstants() {
         // No instances allowed

metafacture-biblio/src/main/java/org/metafacture/biblio/pica/PicaDecoder.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2016 Christoph Böhme
+ * Copyright 2016, 2019 Christoph Böhme and hbz
  *
  * Licensed under the Apache License, Version 2.0 the "License";
  * you may not use this file except in compliance with the License.
@@ -32,9 +32,11 @@
  * containing multiple records must be split into individual records before
  * passing it to {@code PicaDecoder}.
  * <p>
- * The parser is designed to accept any string as valid input and to parse pica
- * plain format as well as normalised pica. To achieve this, the parser behaves
- * as following:
+ * The parser is designed to accept any string as valid input and to parse
+ * pica+ in its two serialization forms:
+ * as non-normalized and as normalized.
+ * To achieve this, the parser behaves as following when parsing:
+ * normalized pica+:
  * <ul>
  *   <li>The parser assumes that the input starts with a field name.
  *
@@ -56,6 +58,26 @@
  *   that field names, subfields, subfield names or subfield values can be
  *   empty.
  * </ul>
+ *  * non-normalized pica+:
+ * <ul>
+ *   <li>The parser assumes that the input starts with a field name.
+ *
+ *   <li>The field name and the first subfield are separated by a subfield
+ *   marker ($).
+ *
+ *   <li>Fields are separated by record markers (&#92;n) or field end
+ *   markers (&#92;n).
+ *
+ *   <li>Subfields are separated by subfield markers ($).
+ *
+ *   <li>The first character of a subfield is the name of the subfield
+ *
+ *   <li>The parser assumes that the end of the input marks the end of the
+ *   current field and the end of the record.
+ *
+ *   <li>As multiple fields and subfields are not empty in non-normailzed pica+
+ *   they are just treated like anything else.
+ * </ul>
  * Please note that the record marker is treated as a field delimiter and not
  * as a record delimiter. Records need to be separated prior to parsing them.
  * <p>
@@ -69,7 +91,8 @@
  *
  *   <li>Subfields which only have a name but no value are always parsed.
  *
- *   <li>Unnamed fields are only parsed if the contain not-ignored subfields.
+ *   <li>In normalized pica+ unnamed fields are only parsed if they contain
+ *    not-ignored subfields. In Non-normalized pica+ unnamed fields don't exist.
  *
  *   <li>Named fields containing none or only ignored subfields are only parsed
  *   if {@link #setSkipEmptyFields(boolean)} is set to false otherwise they are
@@ -85,7 +108,7 @@
  * {@link #setTrimFieldNames(boolean)} to false.
  * <p>
  * The record id emitted with the <i>start-record</i> event is extracted from
- * one of the following pica fields:
+ * one of the following non-normalized pica+ fields:
  * <ul>
  *   <li><i>003&#64; $0</i>
  *   <li><i>107F $0</i>
@@ -97,7 +120,7 @@
  * found in the record a {@link MissingIdException} is thrown otherwise the
  * record identifier is an empty string.
  * <p>
- * For example, when run on the input
+ * For example, when run on this input in its normalized serialization form:
  * <pre>
  * 003&#64; &#92;u001f01234&#92;u001e
  * 028A &#92;u001faAndy&#92;u001fdWarhol&#92;u001e
@@ -120,6 +143,7 @@
  * support other pica encodings.
  *
  * @author Christoph Böhme
+ * @author Pascal Christoph (dr0i)
  *
  */
 @Description("Parses pica+ records. The parser only parses single records. " +
@@ -131,14 +155,11 @@
 public final class PicaDecoder
         extends DefaultObjectPipe<String, StreamReceiver> {
 
-    private static final String START_MARKERS ="(?:^|" + PicaConstants.FIELD_MARKER +
-            "|" + PicaConstants.FIELD_END_MARKER + "|" + PicaConstants.RECORD_MARKER + ")";
-    private static final Pattern ID_FIELDS_PATTERN = Pattern.compile(
-            START_MARKERS + "(?:003@|203@(?:/..+)?|107F) " + PicaConstants.SUBFIELD_MARKER + "0");
-
+    private static String START_MARKERS;
+    private static Pattern ID_FIELDS_PATTERN;
     private static final int BUFFER_SIZE = 1024 * 1024;
 
-    private final Matcher idFieldMatcher = ID_FIELDS_PATTERN.matcher("");
+    private Matcher idFieldMatcher;
     private final StringBuilder idBuilder = new StringBuilder();
     private final PicaParserContext parserContext = new PicaParserContext();
 
@@ -147,6 +168,38 @@ public final class PicaDecoder
 
     private boolean ignoreMissingIdn;
 
+    public PicaDecoder() {
+        makeConstants();
+    }
+
+    public PicaDecoder(boolean normalized) {
+        setNormalizedSerialization(normalized);
+        makeConstants();
+    }
+    /**
+     * Controls wether the input is serialzed as normalized or non-normalized
+     * pica+. As the default "normalized" is assumed.
+     *
+     * @param normalized if true, the input is treated as "normalized" pica+ ;
+     *                   if false, it's treated as non-normalized serialized.
+     */
+    public void setNormalizedSerialization(boolean normalized) {
+        if (normalized)
+            PicaConstants.setNormalizedSerialization();
+        else
+            PicaConstants.setNonNormalizedSerialization();
+        makeConstants();
+    }
+    private void makeConstants() {
+        START_MARKERS = "(?:^|" + PicaConstants.FIELD_MARKER + "|"
+                + PicaConstants.FIELD_END_MARKER + "|"
+                + PicaConstants.RECORD_MARKER + "|.*\n" + ")";
+        ID_FIELDS_PATTERN = Pattern
+                .compile(START_MARKERS + "(?:003@|203@(?:/..+)?|107F) "
+                        + " ?(\\" + PicaConstants.SUBFIELD_MARKER + "|"
+                        + PicaConstants.SUBFIELD_MARKER + ")0");
+        idFieldMatcher = ID_FIELDS_PATTERN.matcher("");
+    }
     /**
      * Controls whether records having no record id are reported as faulty. By
      * default such records are reported by the {@code PicaDecoder} by throwing

metafacture-biblio/src/main/java/org/metafacture/biblio/pica/PicaParserState.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2016 Christoph Böhme
+ * Copyright 2016,2019 Christoph Böhme and hbz
  *
  * Licensed under the Apache License, Version 2.0 the "License";
  * you may not use this file except in compliance with the License.
@@ -25,32 +25,30 @@
  * The parser ignores spaces in field names. They are not included in the
  * field name.
  *
- * Empty subfields are skipped. For instance, parsing the following input
- * would NOT produce an empty literal: 003@ \u001f\u001e. The parser also
+ * Empty subfields are skipped. For instance, parsing the following normalized 
+ * pica+ would NOT produce an empty literal: 003@ \u001f\u001e. The parser also
  * skips unnamed fields without any subfields.
  *
  * @author Christoph Böhme
- *
+ * @author Pascal Christoph (dr0i)
+ * 
  */
 enum PicaParserState {
 
     FIELD_NAME {
         @Override
         protected PicaParserState parseChar(final char ch, final PicaParserContext ctx) {
             final PicaParserState next;
-            switch (ch) {
-            case PicaConstants.RECORD_MARKER:
-            case PicaConstants.FIELD_MARKER:
-            case PicaConstants.FIELD_END_MARKER:
+            if(ch==PicaConstants.RECORD_MARKER ||
+               ch==PicaConstants.FIELD_MARKER ||
+               ch==PicaConstants.FIELD_END_MARKER){
                 ctx.emitStartEntity();
                 ctx.emitEndEntity();
                 next = FIELD_NAME;
-                break;
-            case PicaConstants.SUBFIELD_MARKER:
+            }else if(ch==PicaConstants.SUBFIELD_MARKER){
                 ctx.emitStartEntity();
                 next = SUBFIELD_NAME;
-                break;
-            default:
+            }else{
                 ctx.appendText(ch);
                 next = this;
             }
@@ -67,17 +65,14 @@ protected void endOfInput(final PicaParserContext ctx) {
         @Override
         protected PicaParserState parseChar(final char ch, final PicaParserContext ctx) {
             final PicaParserState next;
-            switch (ch) {
-            case PicaConstants.RECORD_MARKER:
-            case PicaConstants.FIELD_MARKER:
-            case PicaConstants.FIELD_END_MARKER:
+            if(ch==PicaConstants.RECORD_MARKER ||
+               ch==PicaConstants.FIELD_MARKER ||
+               ch==PicaConstants.FIELD_END_MARKER){
                 ctx.emitEndEntity();
                 next = FIELD_NAME;
-                break;
-            case PicaConstants.SUBFIELD_MARKER:
+            }else if(ch==PicaConstants.SUBFIELD_MARKER)
                 next = this;
-                break;
-            default:
+            else{
                 ctx.setSubfieldName(ch);
                 next = SUBFIELD_VALUE;
             }
@@ -93,19 +88,16 @@ protected void endOfInput(final PicaParserContext ctx) {
         @Override
         protected PicaParserState parseChar(final char ch, final PicaParserContext ctx) {
             final PicaParserState next;
-            switch (ch) {
-            case PicaConstants.RECORD_MARKER:
-            case PicaConstants.FIELD_MARKER:
-            case PicaConstants.FIELD_END_MARKER:
+            if(ch==PicaConstants.RECORD_MARKER ||
+               ch==PicaConstants.FIELD_MARKER ||
+               ch==PicaConstants.FIELD_END_MARKER){
                 ctx.emitLiteral();
                 ctx.emitEndEntity();
                 next = FIELD_NAME;
-                break;
-            case PicaConstants.SUBFIELD_MARKER:
+            }else if(ch==PicaConstants.SUBFIELD_MARKER){
                 ctx.emitLiteral();
                 next = SUBFIELD_NAME;
-                break;
-            default:
+            }else{
                 ctx.appendText(ch);
                 next = this;
             }
@@ -122,5 +114,4 @@ protected void endOfInput(final PicaParserContext ctx) {
     protected abstract PicaParserState parseChar(final char ch, final PicaParserContext ctx);
 
     protected abstract void endOfInput(final PicaParserContext ctx);
-
 }

metafacture-biblio/src/test/java/org/metafacture/biblio/pica/PicaDecoderTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2016 Christoph Böhme
+ * Copyright 2016-2019 Christoph Böhme and hbz
  *
  * Licensed under the Apache License, Version 2.0 the "License";
  * you may not use this file except in compliance with the License.
@@ -13,6 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 package org.metafacture.biblio.pica;
 
 import static org.mockito.Mockito.inOrder;
@@ -32,6 +33,7 @@
  * Tests for class {@link PicaDecoder}.
  *
  * @author Christoph Böhme
+ * @author Pascal Christoph (dr0i)
  *
  */
 public final class PicaDecoderTest {
@@ -50,6 +52,10 @@ public final class PicaDecoderTest {
     private static final String SUBFIELD_MARKER = "\u001f";
     private static final String FIELD_END_MARKER = "\n";
 
+    private static final String NONNORMALIZED_RECORD_MARKER = "\n";
+    private static final String NONNORMALIZED_SUBFIELD_MARKER = "$";
+    private static final String NONNORMALIZED_FIELD_END_MARKER = "\n";
+
     private static final String FIELD_001AT_0_TEST = "001@ " + SUBFIELD_MARKER + "0test";
     private static final String FIELD_003AT_0_ID = "003@ " + SUBFIELD_MARKER + "0" + RECORD_ID;
     private static final String FIELD_107F_0_ID = "107F " + SUBFIELD_MARKER + "0" + RECORD_ID;
@@ -59,6 +65,9 @@ public final class PicaDecoderTest {
     private static final String FIELD_021A_A_UEBER = "021A " + SUBFIELD_MARKER + "a" + COMPOSED_UTF8;
     private static final String FIELD_028A = ENTITY_028A + " ";
 
+    private static final String NONNORMALIZED_FIELD_001AT_0_TEST = "001@ " + NONNORMALIZED_SUBFIELD_MARKER + "0test";
+    private static final String NONNORMALIZED_FIELD_003AT_0_ID = "003@ " + NONNORMALIZED_SUBFIELD_MARKER + "0" + RECORD_ID;
+
     private PicaDecoder picaDecoder;
 
     @Mock
@@ -562,6 +571,24 @@ public void shouldNotTrimWhitespaceInFieldNamesIfConfigured() {
         verify(receiver).startEntity("  fieldname  ");
     }
 
+    @Test
+    public void nonNormalizedPica() {
+        picaDecoder.setNormalizedSerialization(false);
+        picaDecoder.process(
+            NONNORMALIZED_FIELD_001AT_0_TEST  +
+            NONNORMALIZED_FIELD_END_MARKER +
+            NONNORMALIZED_FIELD_003AT_0_ID +
+            NONNORMALIZED_RECORD_MARKER);
+        try {
+            verify(receiver).startEntity("001@");
+            verify(receiver).literal("0", "test");
+            verify(receiver).startEntity("003@");
+            verify(receiver).literal("0", "2809");
+        } finally { //ensure reset to the default used by the other tests
+            picaDecoder.setNormalizedSerialization(true);
+        }
+    }
+
     private void verify003At0ID(final InOrder ordered) {
         ordered.verify(receiver).startEntity("003@");
         ordered.verify(receiver).literal("0", RECORD_ID);