Process document recursively

Set generated record ID, only process content of leaf nodes

See #312

Author: fsteeg

metafacture-html/src/main/java/org/metafacture/html/HtmlDecoder.java

@@ -17,11 +17,13 @@
 
 import java.io.IOException;
 import java.io.Reader;
+import java.util.UUID;
 
 import org.apache.commons.io.IOUtils;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Attribute;
 import org.jsoup.nodes.Attributes;
+import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.metafacture.framework.FluxCommand;
 import org.metafacture.framework.StreamReceiver;
@@ -46,21 +48,31 @@ public class HtmlDecoder extends DefaultObjectPipe<Reader, StreamReceiver> {
     public void process(final Reader reader) {
         try {
             StreamReceiver receiver = getReceiver();
-            receiver.startRecord(null);
-            String html = IOUtils.toString(reader);
-            for (Element element : Jsoup.parse(html).getAllElements()) {
-                receiver.startEntity(element.nodeName());
-                Attributes attributes = element.attributes();
-                for (Attribute attribute : attributes) {
-                    receiver.literal(attribute.getKey(), attribute.getValue());
-                }
-                String text = element.text().trim();
-                receiver.literal("value", text.isEmpty() ? element.data() : text);
-                receiver.endEntity();
-            }
+            receiver.startRecord(UUID.randomUUID().toString());
+            Document document = Jsoup.parse(IOUtils.toString(reader));
+            process(document, receiver);
             receiver.endRecord();
         } catch (IOException e) {
             e.printStackTrace();
         }
     }
+
+    private void process(Element parent, StreamReceiver receiver) {
+        for (Element element : parent.children()) {
+            receiver.startEntity(element.nodeName());
+            Attributes attributes = element.attributes();
+            for (Attribute attribute : attributes) {
+                receiver.literal(attribute.getKey(), attribute.getValue());
+            }
+            if (element.children().isEmpty()) {
+                String text = element.text().trim();
+                String value = text.isEmpty() ? element.data() : text;
+                if (!value.isEmpty()) {
+                    receiver.literal("value", value);
+                }
+            }
+            process(element, receiver);
+            receiver.endEntity();
+        }
+    }
 }

metafacture-html/src/test/java/org/metafacture/html/HtmlDecoderTest.java

@@ -16,6 +16,7 @@
 package org.metafacture.html;
 
 import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.times;
 
 import java.io.StringReader;
 
@@ -50,23 +51,28 @@ public void setup() {
     public void htmlElementsAsEntities() {
         htmlDecoder.process(new StringReader("<h1>Header</h1><p>Paragraph</p>"));
         final InOrder ordered = inOrder(receiver);
+        ordered.verify(receiver).startEntity("html");
+        ordered.verify(receiver).startEntity("head");
+        ordered.verify(receiver).endEntity();
+        ordered.verify(receiver).startEntity("body");
         ordered.verify(receiver).startEntity("h1");
         ordered.verify(receiver).literal("value", "Header");
         ordered.verify(receiver).endEntity();
         ordered.verify(receiver).startEntity("p");
         ordered.verify(receiver).literal("value", "Paragraph");
-        ordered.verify(receiver).endEntity();
+        ordered.verify(receiver, times(3)).endEntity();
     }
 
     @Test
     public void nestedEntities() {
-        htmlDecoder.process(new StringReader("<ul><li>Item</li><ul>"));
+        htmlDecoder.process(new StringReader("<ul><li>Item</li></ul>"));
         final InOrder ordered = inOrder(receiver);
         ordered.verify(receiver).startEntity("ul");
         ordered.verify(receiver).startEntity("li");
         ordered.verify(receiver).literal("value", "Item");
-        ordered.verify(receiver).endEntity();
-        ordered.verify(receiver).endEntity();
+        // elements above plus body, html
+        ordered.verify(receiver, times(4)).endEntity();
+
     }
 
     @Test
@@ -76,17 +82,19 @@ public void htmlAttributesAsLiterals() {
         ordered.verify(receiver).startEntity("p");
         ordered.verify(receiver).literal("class", "lead");
         ordered.verify(receiver).literal("value", "Text");
-        ordered.verify(receiver).endEntity();
+        // elements above plus body, html
+        ordered.verify(receiver, times(3)).endEntity();
     }
-    
+
     @Test
     public void htmlScriptElementData() {
         htmlDecoder.process(new StringReader("<script type=application/ld+json>{\"id\":\"theId\"}</script>"));
         final InOrder ordered = inOrder(receiver);
         ordered.verify(receiver).startEntity("script");
         ordered.verify(receiver).literal("type", "application/ld+json");
         ordered.verify(receiver).literal("value", "{\"id\":\"theId\"}");
-        ordered.verify(receiver).endEntity();
+        // elements above plus body, html
+        ordered.verify(receiver, times(4)).endEntity();
     }
 
 }