Merge branch 'master' of ssh://github.com/culturegraph/metafacture-core

Author: mgeipel

src/main/java/org/culturegraph/mf/stream/pipe/Utf8Normalizer.java

@@ -0,0 +1,44 @@
+/*
+ *  Copyright 2013 Deutsche Nationalbibliothek
+ *
+ *  Licensed under the Apache License, Version 2.0 the "License";
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.culturegraph.mf.stream.pipe;
+
+import java.text.Normalizer;
+import java.text.Normalizer.Form;
+
+import org.culturegraph.mf.framework.DefaultObjectPipe;
+import org.culturegraph.mf.framework.ObjectReceiver;
+import org.culturegraph.mf.framework.annotations.Description;
+import org.culturegraph.mf.framework.annotations.In;
+import org.culturegraph.mf.framework.annotations.Out;
+
+/**
+ * Normalises diacritics in UTF-8 encoded strings.
+ * 
+ * @author Christoph Böhme
+ *
+ */
+@Description("Normalizes diacritics UTF-8 encoded strings.")
+@In(String.class)
+@Out(String.class)
+public final class Utf8Normalizer extends
+		DefaultObjectPipe<String, ObjectReceiver<String>> {
+
+	@Override
+	public void process(final String str) {
+		getReceiver().process(Normalizer.normalize(str, Form.NFC));
+	}
+	
+}

src/main/resources/flux-commands.properties

@@ -78,6 +78,7 @@ draw-uniform-sample org.culturegraph.mf.stream.pipe.UniformSampler
 
 catch-object-exception org.culturegraph.mf.stream.pipe.ObjectExceptionCatcher
 
+normalize-utf8 org.culturegraph.mf.stream.pipe.Utf8Normalizer
 
 morph	org.culturegraph.mf.morph.Metamorph
 filter	org.culturegraph.mf.stream.pipe.Filter

src/test/java/org/culturegraph/mf/morph/functions/NormalizeUTF8Test.java

@@ -13,26 +13,31 @@
  *  See the License for the specific language governing permissions and
  *  limitations under the License.
  */
-package org.culturegraph.mf.morph.functions;
-
-import static org.junit.Assert.assertEquals;
-
-import org.culturegraph.mf.morph.functions.NormalizeUTF8;
-import org.junit.Test;
-
-
-/**
- * @author Christoph Böhme <[email protected]>
- *
- */
-public final class NormalizeUTF8Test {
-
-	private static final String INPUT_STR = "Bauer, Sigmund: Über den Einfluß der Ackergeräthe auf den Reinertrag.";
-	private static final String OUTPUT_STR = "Bauer, Sigmund: Über den Einfluß der Ackergeräthe auf den Reinertrag.";
-	
-	@Test
-	public void testProcess() {
-		final NormalizeUTF8 normalize = new NormalizeUTF8();
-		assertEquals("Normalization incorrect", OUTPUT_STR, normalize.process(INPUT_STR));
-	}
-}
+package org.culturegraph.mf.morph.functions;
+
+import static org.junit.Assert.assertEquals;
+
+import org.culturegraph.mf.morph.functions.NormalizeUTF8;
+import org.junit.Test;
+
+
+/**
+ * @author Christoph Böhme <[email protected]>
+ *
+ */
+public final class NormalizeUTF8Test {
+
+	// The umlauts in this string are composed of two characters (u and ", e.g.):
+	private static final String INPUT_STR = 
+			"Bauer, Sigmund: Über den Einfluß der Ackergeräthe auf den Reinertrag.";
+	
+	// The umlauts in this string are individual characters:
+	private static final String OUTPUT_STR = 
+			"Bauer, Sigmund: Über den Einfluß der Ackergeräthe auf den Reinertrag.";
+	
+	@Test
+	public void testProcess() {
+		final NormalizeUTF8 normalize = new NormalizeUTF8();
+		assertEquals("Normalization incorrect", OUTPUT_STR, normalize.process(INPUT_STR));
+	}
+}

src/test/java/org/culturegraph/mf/stream/pipe/Utf8NormalizerTest.java

@@ -0,0 +1,53 @@
+/*
+ *  Copyright 2013 Deutsche Nationalbibliothek
+ *
+ *  Licensed under the Apache License, Version 2.0 the "License";
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.culturegraph.mf.stream.pipe;
+
+import static org.junit.Assert.assertEquals;
+
+import org.culturegraph.mf.framework.DefaultObjectReceiver;
+import org.junit.Test;
+
+/**
+ * Tests {@link Utf8Normalizer}.
+ * 
+ * @author Christoph Böhme
+ *
+ */
+public final class Utf8NormalizerTest {
+
+	// The umlauts in this string are composed of two characters (u and ", e.g.):
+	private static final String INPUT_STR = 
+			"Bauer, Sigmund: Über den Einfluß der Ackergeräthe auf den Reinertrag.";
+	
+	// The umlauts in this string are individual characters:
+	private static final String OUTPUT_STR = 
+			"Bauer, Sigmund: Über den Einfluß der Ackergeräthe auf den Reinertrag.";
+	
+	@Test
+	public void testNormalization() {
+		final Utf8Normalizer normalizer = new Utf8Normalizer();
+		
+		normalizer.setReceiver(new DefaultObjectReceiver<String>() {
+			@Override
+			public void process(final String obj) {
+				assertEquals(OUTPUT_STR, obj);
+			}
+		});
+		
+		normalizer.process(INPUT_STR);
+	}
+
+}