Basic HtmlReader with html-to-xml flux command

Parse HTML with jsoup, write XML. See example in test.

See #312

Author: fsteeg

metafacture-html/build.gradle

@@ -0,0 +1,30 @@
+/*
+ * Copyright 2020 Fabian Steeg, hbz
+ *
+ * Licensed under the Apache License, Version 2.0 the "License";
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ext.mavenName = 'Metafacture HTML'
+description = 'Modules for processing HTML documents'
+
+dependencies {
+  api project(':metafacture-framework')
+  implementation project(':metafacture-commons')
+  implementation 'org.slf4j:slf4j-api:1.7.21'
+  implementation 'org.apache.commons:commons-compress:1.12'
+  implementation 'commons-io:commons-io:2.6'
+  implementation 'org.jsoup:jsoup:1.12.1'
+  testImplementation 'junit:junit:4.12'
+  testImplementation 'org.mockito:mockito-core:2.5.5'
+  testRuntimeOnly 'org.slf4j:slf4j-simple:1.7.21'
+}

metafacture-html/src/main/java/org/metafacture/html/HtmlReader.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright 2020 Fabian Steeg, hbz
+ *
+ * Licensed under the Apache License, Version 2.0 the "License";
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.metafacture.html;
+
+import java.io.IOException;
+import java.io.Reader;
+
+import org.apache.commons.io.IOUtils;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.metafacture.framework.FluxCommand;
+import org.metafacture.framework.ObjectReceiver;
+import org.metafacture.framework.annotations.Description;
+import org.metafacture.framework.annotations.In;
+import org.metafacture.framework.annotations.Out;
+import org.metafacture.framework.helpers.DefaultObjectPipe;
+
+/**
+ * Parses HTML to X(HT)ML
+ *
+ * @author Fabian Steeg
+ */
+@Description("Parses HTML to X(HT)ML")
+@In(Reader.class)
+@Out(String.class)
+@FluxCommand("html-to-xml")
+public class HtmlReader extends DefaultObjectPipe<Reader, ObjectReceiver<String>> {
+    @Override
+    public void process(final Reader reader) {
+        try {
+            Document document = Jsoup.parse(IOUtils.toString(reader));
+            document.outputSettings().prettyPrint(false).syntax(Document.OutputSettings.Syntax.xml);
+            getReceiver().process(document.html());
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+}

metafacture-html/src/main/resources/flux-commands.properties

@@ -0,0 +1,16 @@
+#
+# Copyright 2020 Fabian Steeg, hbz
+#
+# Licensed under the Apache License, Version 2.0 the "License";
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+html-to-xml org.metafacture.html.HtmlReader

metafacture-html/src/test/java/org/metafacture/html/HtmlReaderTest.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2020 Fabian Steeg, hbz
+ *
+ * Licensed under the Apache License, Version 2.0 the "License";
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.metafacture.html;
+
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+
+import java.io.StringReader;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.metafacture.framework.ObjectReceiver;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+/**
+ * Tests for {@link HtmlReader}.
+ *
+ * @author Fabian Steeg
+ *
+ */
+public final class HtmlReaderTest {
+
+    private static final StringReader IN = new StringReader("<html><i>hi");
+    private static final String OUT = "<html><head></head><body><i>hi</i></body></html>";
+
+    private HtmlReader htmlReader;
+
+    @Mock
+    private ObjectReceiver<String> receiver;
+
+    @Before
+    public void setup() {
+        MockitoAnnotations.initMocks(this);
+        htmlReader = new HtmlReader();
+        htmlReader.setReceiver(receiver);
+    }
+
+    @Test
+    public void testShouldProcessRecordsFollowedbySeparator() {
+        htmlReader.process(IN);
+        verify(receiver).process(OUT);
+        verifyNoMoreInteractions(receiver);
+    }
+
+    @After
+    public void cleanup() {
+        htmlReader.closeStream();
+    }
+}

metafacture-html/src/test/resources/simplelogger.properties

@@ -0,0 +1,15 @@
+# Copyright 2020 Fabian Steeg, hbz
+#
+# Licensed under the Apache License, Version 2.0 the "License";
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+org.slf4j.simpleLogger.defaultLogLevel = DEBUG

settings.gradle

@@ -28,6 +28,7 @@ include ':metafacture-strings'
 include ':metafacture-formeta'
 include ':metafacture-formatting'
 include ':metafacture-xml'
+include ':metafacture-html'
 include ':metafacture-triples'
 include ':metafacture-statistics'
 include ':metafacture-io'