Merge pull request #117 from ah641054/eliminate-duplicate-triples

Adds a module to filter consecutive duplicated objects.

Author: cboehme

src/main/java/org/culturegraph/mf/stream/pipe/DuplicateObjectFilter.java

@@ -0,0 +1,57 @@
+/*
+ *  Copyright 2013 Deutsche Nationalbibliothek
+ *
+ *  Licensed under the Apache License, Version 2.0 the "License";
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.culturegraph.mf.stream.pipe;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.culturegraph.mf.framework.DefaultObjectPipe;
+import org.culturegraph.mf.framework.ObjectReceiver;
+import org.culturegraph.mf.framework.annotations.Description;
+import org.culturegraph.mf.framework.annotations.In;
+import org.culturegraph.mf.framework.annotations.Out;
+import org.culturegraph.mf.types.Triple;
+
+/**
+ * Filters consecutive duplicated data objects. 
+ * 
+ * @param <T> object type
+ *
+ * @author Alexander Haffner
+ *
+ */
+@Description("Filters consecutive duplicated data objects.")
+@In(Object.class)
+@Out(Object.class)
+public final class DuplicateObjectFilter<T> extends
+		DefaultObjectPipe<T, ObjectReceiver<T>> {
+
+	private T lastObj;
+	
+	@Override
+	public void process(final T obj) {
+		if (!obj.equals(lastObj)) {
+			lastObj = obj;
+			getReceiver().process(obj);
+		}		
+	}
+	
+	@Override
+	protected void onResetStream() {
+		lastObj = null;
+	}
+
+}

src/main/resources/flux-commands.properties

@@ -62,6 +62,8 @@ flatten	org.culturegraph.mf.stream.pipe.StreamFlattener
 template org.culturegraph.mf.stream.converter.ObjectTemplate
 add-preamble-epilogue org.culturegraph.mf.stream.converter.PreambleEpilogueAdder
 
+filter-duplicate-objects org.culturegraph.mf.stream.pipe.DuplicateObjectFilter
+
 object-tee org.culturegraph.mf.stream.pipe.ObjectTee
 stream-tee org.culturegraph.mf.stream.pipe.StreamTee
 

src/test/java/org/culturegraph/mf/stream/pipe/DuplicateObjectFilterTest.java

@@ -0,0 +1,55 @@
+/**
+ * 
+ */
+package org.culturegraph.mf.stream.pipe;
+
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+
+import org.culturegraph.mf.framework.ObjectReceiver;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+
+/**
+ * Tests for {@link DuplicateObjectFilter}.
+ * 
+ * @author Alexander Haffner
+ *
+ */
+public final class DuplicateObjectFilterTest {
+
+	private static final String OBJECT1 = "Object 1";
+	private static final String OBJECT2 = "Object 2";
+
+	private DuplicateObjectFilter<String> duplicateObjectFilter;
+	
+	@Mock
+	private ObjectReceiver<String> receiver;
+	
+	@Before
+	public void setup() {
+		MockitoAnnotations.initMocks(this);
+		duplicateObjectFilter = new DuplicateObjectFilter<String>();
+		duplicateObjectFilter.setReceiver(receiver);
+	}
+	
+	@After
+	public void cleanup() {
+		duplicateObjectFilter.closeStream();
+	}
+	
+	@Test
+	public void testShouldEliminateDuplicateObjects() {
+		duplicateObjectFilter.process(OBJECT1);
+		duplicateObjectFilter.process(OBJECT1);
+		duplicateObjectFilter.process(OBJECT2);
+		
+		verify(receiver).process(OBJECT1);
+		verify(receiver).process(OBJECT2);
+		verifyNoMoreInteractions(receiver);
+	}
+
+}