Merge pull request #70 from cboehme/triple-object-retriever

Added TripleObjectRetriever module

Author: cboehme

src/main/java/org/culturegraph/mf/stream/pipe/TripleObjectRetriever.java

@@ -0,0 +1,93 @@
+/*
+ *  Copyright 2013 Deutsche Nationalbibliothek
+ *
+ *  Licensed under the Apache License, Version 2.0 the "License";
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.culturegraph.mf.stream.pipe;
+
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLConnection;
+
+import org.apache.commons.io.IOUtils;
+import org.culturegraph.mf.exceptions.MetafactureException;
+import org.culturegraph.mf.framework.DefaultObjectPipe;
+import org.culturegraph.mf.framework.ObjectReceiver;
+import org.culturegraph.mf.framework.annotations.Description;
+import org.culturegraph.mf.framework.annotations.In;
+import org.culturegraph.mf.framework.annotations.Out;
+import org.culturegraph.mf.types.Triple;
+import org.culturegraph.mf.types.Triple.ObjectType;
+
+/**
+ * Uses the object value of the triple as a URL and emits a new triple
+ * in which the object value is replaced with the contents of the resource
+ * identified by the URL.
+ * 
+ * @author Christoph Böhme
+ */
+@Description("Uses the object value of the triple as a URL and emits a new triple "
+		+ "in which the object value is replaced with the contents of the resource "
+		+ "identified by the URL.")
+@In(Triple.class)
+@Out(Triple.class)
+public final class TripleObjectRetriever 
+		extends DefaultObjectPipe<Triple, ObjectReceiver<Triple>> {
+
+	private String defaultEncoding = "UTF-8";
+	
+	/**
+	 * Returns the default encoding used when no encoding is
+	 * provided by the server. The default setting is UTF-8.
+	 * 
+	 * @return current default setting
+	 */
+	public String getDefaultEncoding() {
+		return defaultEncoding;
+	}
+
+	/**
+	 * Sets the default encoding to use when no encoding is 
+	 * provided by the server. The default setting is UTF-8.
+	 * 
+	 * @param defaultEncoding new default encoding
+	 */
+	public void setDefaultEncoding(final String defaultEncoding) {
+		this.defaultEncoding = defaultEncoding;
+	}
+
+	@Override
+	public void process(final Triple triple) {
+		assert !isClosed();
+		
+		if (triple.getObjectType() != ObjectType.STRING) {
+			return;
+		}
+
+		final String objectValue;
+		try {
+			final URL url = new URL(triple.getObject());
+			final URLConnection con = url.openConnection();
+			String enc = con.getContentEncoding();
+			if (enc == null) {
+				enc = defaultEncoding;
+			}
+			objectValue = IOUtils.toString(con.getInputStream(), enc);
+		} catch (IOException e) {
+			throw new MetafactureException(e);
+		}
+
+		getReceiver().process(new Triple(triple.getSubject(), triple.getPredicate(), objectValue));
+	}
+	
+}

src/test/java/org/culturegraph/mf/stream/pipe/TripleObjectRetrieverTest.java

@@ -0,0 +1,101 @@
+/*
+ *  Copyright 2013 Deutsche Nationalbibliothek
+ *
+ *  Licensed under the Apache License, Version 2.0 the "License";
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.culturegraph.mf.stream.pipe;
+
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.Writer;
+
+import org.apache.commons.io.IOUtils;
+import org.culturegraph.mf.framework.ObjectReceiver;
+import org.culturegraph.mf.types.Triple;
+import org.culturegraph.mf.types.Triple.ObjectType;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+/**
+ * Tests for {@link TripleObjectRetriever}.
+ * 
+ * @author Christoph Böhme
+ */
+public final class TripleObjectRetrieverTest {
+
+	private static final String SUBJECT = "subject";
+	private static final String PREDICATE = "predicate";
+	private static final String OBJECT_VALUE = "object-data";
+	private static final String ENTITY = "{l=v}";
+
+	private TripleObjectRetriever tripleObjectRetriever;
+	
+	@Mock
+	private ObjectReceiver<Triple> receiver;
+	
+	// NO CHECKSTYLE VisibilityModifier|DeclarationOrder FOR 3 LINES:
+	// JUnit requires rules to be public
+	@Rule
+	public TemporaryFolder tempFolder = new TemporaryFolder();
+
+	private String objectUrl;
+	
+	@Before
+	public void setup() throws IOException {
+		MockitoAnnotations.initMocks(this);
+		
+		tripleObjectRetriever = new TripleObjectRetriever();
+		tripleObjectRetriever.setReceiver(receiver);
+		
+		objectUrl = createObjectResource(OBJECT_VALUE);
+	}
+	
+	@After
+	public void cleanup() {
+		tripleObjectRetriever.closeStream();
+	}
+	
+	@Test
+	public void testShouldReplaceObjectValueWithResourceContentRetrievedFromUrl() {
+		tripleObjectRetriever.process(new Triple(SUBJECT, PREDICATE, objectUrl));
+		
+		verify(receiver).process(new Triple(SUBJECT, PREDICATE, OBJECT_VALUE));
+	}
+	
+	@Test
+	public void testShouldSkipTriplesWithObjectTypeEntity() {
+		tripleObjectRetriever.process(new Triple(SUBJECT, PREDICATE, ENTITY, ObjectType.ENTITY));
+		
+		verifyZeroInteractions(receiver);
+	}
+	
+	private String createObjectResource(final String contents) throws IOException {
+		final File file = tempFolder.newFile();
+		
+		final Writer writer = new FileWriter(file);
+		IOUtils.write(contents, writer);
+		writer.close();
+		
+		return file.toURI().toURL().toString();
+	}
+
+}