Merge pull request #1414 from dkpro/bugfix/1413-CoNLL-U-export-broken-if-text-contains-line-breaks

#1413 - CoNLL-U export broken if text contains line breaks

Author: reckart

dkpro-core-io-conll-asl/src/main/java/org/dkpro/core/io/conll/ConllUWriter.java

@@ -169,7 +169,11 @@ private void convert(JCas aJCas, PrintWriter aOut)
                 aOut.printf("# %s = %s\n", ConllUReader.META_SEND_ID, sentence.getId());
             }
             if (writeTextHeader) {
-                aOut.printf("# %s = %s\n", ConllUReader.META_TEXT, sentence.getCoveredText());
+                String sentenceText = sentence.getCoveredText();
+                // CoNLL-U does not support line breaks in the sentence text, so we need to replace
+                // such characters.
+                sentenceText = StringUtils.replaceChars(sentenceText, "\n\r", "  ");
+                aOut.printf("# %s = %s\n", ConllUReader.META_TEXT, sentenceText);
             }
 
             // Tokens

dkpro-core-io-conll-asl/src/test/java/org/dkpro/core/io/conll/ConllUWriterTest.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright 2019
+ * Ubiquitous Knowledge Processing (UKP) Lab
+ * Technische Universität Darmstadt
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.dkpro.core.io.conll;
+
+import static org.apache.commons.io.FileUtils.readFileToString;
+import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngine;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.File;
+
+import org.apache.uima.analysis_engine.AnalysisEngine;
+import org.apache.uima.fit.factory.JCasFactory;
+import org.apache.uima.jcas.JCas;
+import org.dkpro.core.testing.DkproTestContext;
+import org.junit.Rule;
+import org.junit.Test;
+
+import de.tudarmstadt.ukp.dkpro.core.api.metadata.type.DocumentMetaData;
+import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence;
+import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token;
+
+public class ConllUWriterTest
+{
+    @Test
+    public void thatLineBreaksDoNotBreakTheFormat() throws Exception
+    {
+        File target = testContext.getTestOutputFolder();
+
+        JCas jcas = JCasFactory.createText("Test\ntest.");
+        new Sentence(jcas, 0, 10).addToIndexes();
+        new Token(jcas, 0, 4).addToIndexes();
+        new Token(jcas, 5, 9).addToIndexes();
+        new Token(jcas, 9, 10).addToIndexes();
+
+        DocumentMetaData dmd = DocumentMetaData.create(jcas);
+        dmd.setDocumentId("output");
+
+        AnalysisEngine writer = createEngine(ConllUWriter.class, 
+                ConllUWriter.PARAM_TARGET_LOCATION, target);
+
+        writer.process(jcas);
+
+        String reference = readFileToString(
+                new File("src/test/resources/conll/u_v2/conllu-linebreaks.conll"), "UTF-8").trim();
+        String actual = readFileToString(new File(target, "output.conll"), "UTF-8").trim();
+
+        assertThat(actual).isEqualToNormalizingNewlines(reference);
+    }
+
+    @Rule
+    public DkproTestContext testContext = new DkproTestContext();
+}

dkpro-core-io-conll-asl/src/test/resources/conll/u_v2/conllu-linebreaks.conll

@@ -0,0 +1,5 @@
+# text = Test test.
+1	Test	_	_	_	_	_	_	_	_
+2	test	_	_	_	_	_	_	_	SpaceAfter=No
+3	.	_	_	_	_	_	_	_	_
+